diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 0000000..b5d9cac
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,32 @@
+# Copyright (c) 2015, 2017 YCSB contributors.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+# For more info, see: http://EditorConfig.org
+root = true
+
+[*.java]
+indent_style = space
+indent_size = 2
+continuation_indent_size = 4
+
+[*.md]
+indent_style = space
+indent_size = 2
+continuation_indent_size = 4
+
+[*.xml]
+indent_style = space
+indent_size = 2
+continuation_indent_size = 4
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..aa65bac
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,25 @@
+# ignore compiled byte code
+target
+
+# ignore output files from testing
+output*
+
+# ignore standard Eclipse files
+.project
+.classpath
+.settings
+.checkstyle
+
+# ignore standard IntelliJ files
+.idea/
+*.iml
+*.ipr
+*.iws
+
+# ignore standard Vim and Emacs temp files
+*.swp
+*~
+
+# ignore standard Mac OS X files/dirs
+.DS_Store
+/differentbin/
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..5fb89e9
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,53 @@
+# Copyright (c) 2010 Yahoo! Inc., 2012 - 2015 YCSB contributors.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+
+# more info here about TravisCI and Java projects
+# http://docs.travis-ci.com/user/languages/java/
+
+language: java
+
+jdk:
+ - openjdk8
+ - openjdk11
+ - oraclejdk11
+
+addons:
+ hosts:
+ - myshorthost
+ hostname: myshorthost
+ postgresql: "9.5"
+
+install:
+ - mvn -N io.takari:maven:0.7.7:wrapper -Dmaven=3.6.3
+ - ./mvnw install -q -DskipTests=true
+
+script: ./mvnw test -q
+
+before_script:
+ - psql -c 'CREATE database test;' -U postgres
+ - psql -c 'CREATE TABLE usertable (YCSB_KEY VARCHAR(255) PRIMARY KEY not NULL, YCSB_VALUE JSONB not NULL);' -U postgres -d test
+ - psql -c 'GRANT ALL PRIVILEGES ON DATABASE test to postgres;' -U postgres
+
+# Services to start for tests.
+services:
+ - ignite
+ - mongodb
+ - postgresql
+# temporarily disable riak. failing, docs offline.
+# - riak
+
+# Can't use container based infra because of hosts/hostname
+sudo: true
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..b2e0ee4
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,133 @@
+
+## How To Contribute
+
+As more and more databases are created to handle distributed or "cloud" workloads, YCSB needs contributors to write clients to test them. And of course we always need bug fixes, updates for existing databases and new features to keep YCSB going. Here are some guidelines to follow when digging into the code.
+
+## Project Source
+
+YCSB is located in a Git repository hosted on GitHub at [https://github.com/brianfrankcooper/YCSB](https://github.com/brianfrankcooper/YCSB). To modify the code, fork the main repo into your own GitHub account or organization and commit changes there.
+
+YCSB is written in Java (as most of the new cloud data stores at beginning of the project were written in Java) and is laid out as a multi-module Maven project. You should be able to import the project into your favorite IDE or environment easily. For more details about the Maven layout see the [Guide to Working with Multiple Modules](https://maven.apache.org/guides/mini/guide-multiple-modules.html).
+
+## Licensing
+
+YCSB is licensed under the Apache License, Version 2.0 (APL2). Every file included in the project must include the APL header. For example, each Java source file must have a header similar to the following:
+
+```java
+/**
+ * Copyright (c) 2015-2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+```
+
+When modifying files that already have a license header, please update the year when you made your edits. E.g. change ``Copyright (c) 2010 Yahoo! Inc., 2012 - 2016 YCSB contributors.`` to ``Copyright (c) 2010 Yahoo! Inc., 2012 - 2017 YCSB contributors.`` If the file only has ``Copyright (c) 2010 Yahoo! Inc.``, append the current year as in ``Copyright (c) 2010 Yahoo! Inc., 2017 YCSB contributors.``.
+
+**WARNING**: It should go without saying, but don't copy and paste code from outside authors or sources. If you are a database author and want to copy some example code, it must be APL2 compatible.
+
+Client bindings to non-APL databases are perfectly acceptable, as data stores are meant to be used from all kinds of projects. Just make sure not to copy any code or commit libraries or binaries into the YCSB code base. Link to them in the Maven pom file.
+
+## Issues and Support
+
+To track bugs, feature requests and releases we use GitHub's integrated [Issues](https://github.com/brianfrankcooper/YCSB/issues). If you find a bug or problem, open an issue with a descriptive title and as many details as you can give us in the body (stack traces, log files, etc). Then if you can create a fix, follow the PR guidelines below.
+
+**Note** Before embarking on a code change or DB, search through the existing issues and pull requests to see if anyone is already working on it. Reach out to them if so.
+
+For general support, please use the mailing list hosted (of course) with Yahoo groups at [http://groups.yahoo.com/group/ycsb-users](http://groups.yahoo.com/group/ycsb-users).
+
+## Code Style
+
+A Java coding style guide is enforced via the Maven CheckStyle plugin. We try not to be too draconian with enforcement but the biggies include:
+
+* Whitespaces instead of tabs.
+* Proper Javadocs for methods and classes.
+* Camel case member names.
+* Upper camel case classes and method names.
+* Line length.
+
+CheckStyle will run for pull requests or if you create a package locally so if you just compile and push a commit, you may be surprised when the build fails with a style issue. Just execute ``mvn checkstyle:checkstyle `` before you open a PR and you should avoid any suprises.
+
+## Platforms
+
+Since most data bases aim to support multiple platforms, YCSB aims to run on as many as possible as well. Besides **Linux** and **macOS**, YCSB must compile and run for **Windows**. While not all DBs will run under every platform, the YCSB tool itself must be able to execute on all of these systems and hopefully be able to communicate with remote data stores.
+
+Additionally, YCSB is targeting Java 7 (1.7.0) as its build version as some users are glacially slow moving to Java 8. So please avoid those Lambdas and Streams for now.
+
+## Pull Requests
+
+You've written some amazing code and are excited to share it with the community! It's time to open a PR! Here's what you should do.
+
+* Checkout YCSB's ``master`` branch in your own fork and create a new branch based off of it with a name that is reflective of your work. E.g. ``i123`` for fixing an issue or ``db_xyz`` when working on a binding.
+* Add your changes to the branch.
+* Commit the code and start the commit message with the component you are working on in square braces. E.g. ``[core] Add another format for exporting histograms.`` or ``[hbase12] Fix interrupted exception bug.``.
+* Push to your fork and click the ``Create Pull Request`` button.
+* Wait for the build to complete in the CI pipeline. If it fails with a red X, click through the logs for details and fix any issues and commit your changes.
+* If you have made changes, please flatten the commits so that the commit logs are nice and clean. Just run a ``git rebase -i ``.
+
+After you have opened your PR, a YCSB maintainer will review it and offer constructive feedback via the GitHub review feature. If no one has responded to your PR, please bump the thread by adding comments.
+
+**NOTE**: For maintainers, please get another maintainer to sign off on your changes before merging a PR. And if you're writing code, please do create a PR from your fork, don't just push code directly to the master branch.
+
+## Core, Bindings and Workloads
+
+The main components of the code base include the core library and benchmarking utility, various database client bindings and workload classes and definitions.
+
+### Core
+When working on the core classes, keep in mind the following:
+
+* Do not change the core behavior or operation of the main benchmarking classes (Particularly the Client and Workload classes). YCSB is used all over the place because it's a consistent standard that allows different users to compare results with the same workloads. If you find a way to drastically improve throughput, that's great! But please check with the rest of the maintainers to see if we can add the tweaks without invalidating years of benchmarks.
+* Do not remove or modify measurements. Users may have tooling to parse the outputs so if you take something out, they'll be a wee bit unhappy. Extending or adding measurements is fine (so if you do have tooling, expect additions.)
+* Do not modify existing generators. Again we don't want to invalidate years of benchmarks. Instead, create a new generator or option that can be enabled explicitly (not implicitly!) for users to try out.
+* Utility classes and methods are welcome. But if they're only ever used by a specific database binding, co-locate the code with that binding.
+* Don't change the DB interface if at all possible. Implementations can squeeze all kinds of workloads through the existing interface and while it may be easy to change the bindings included with the source code, some users may have private clients they can't share with the community.
+
+### Bindings and Clients
+
+When a new database is released a *binding* can be created that implements a client communicating with the given data store that will execute YCSB workloads. Details about writing a DB binding can be found on our [GitHub Wiki page](https://github.com/brianfrankcooper/YCSB/wiki/Adding-a-Database). Some development guidelines to follow include:
+
+* Create a new Maven module for your binding. Follow the existing bindings as examples.
+* The module *must* include a README.md file with details such as:
+ * Database setup with links to documentation so that the YCSB benchmarks will execute properly.
+ * Example command line executions (workload selection, etc).
+ * Required and optional properties (e.g. connection strings, behavior settings, etc) along with the default values.
+ * Versions of the database the binding supports.
+* Javadoc the binding and all of the methods. Tell us what it does and how it works.
+
+Because YCSB is a utility to compare multiple data stores, we need each binding to behave similarly by default. That means each data store should enforce the strictest consistency guarantees available and avoid client side buffering or optimizations. This allows users to evaluate different DBs with a common baseline and tough standards.
+
+However you *should* include parameters to tune and improve performance as much as possible to reach those flashy marketing numbers. Just be honest and document what the settings do and what trade-offs are made. (e.g. client side buffering reduces I/O but a crash can lead to data loss).
+
+### Workloads
+
+YCSB began comparing various key/value data stores with simple CRUD operations. However as DBs have become more specialized we've added more workloads for various tasks and would love to have more in the future. Keep the following in mind:
+
+* Make sure more than one publicly available database can handle your workload. It's no fun if only one player is in the game.
+* Use the existing DB interface to pass your data around. If you really need another API, discuss with the maintainers to see if there isn't a workaround.
+* Provide real-world use cases for the workload, not just theoretical idealizations.
\ No newline at end of file
diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/NOTICE.txt b/NOTICE.txt
new file mode 100644
index 0000000..cd1f104
--- /dev/null
+++ b/NOTICE.txt
@@ -0,0 +1,13 @@
+=========================================================================
+NOTICE file for use with, and corresponding to Section 4 of,
+the Apache License, Version 2.0,
+in this case for the YCSB project.
+=========================================================================
+
+ This product includes software developed by
+ Yahoo! Inc. (www.yahoo.com)
+ Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+
+ This product includes software developed by
+ Google Inc. (www.google.com)
+ Copyright (c) 2015 Google Inc. All rights reserved.
diff --git a/README.md b/README.md
index 8b13789..d554d2a 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,157 @@
+
+
+About
+====================================
+
+This is a modification of the Yahoo! Cloud Serving Benchmark (YCSB) that brings a new workload and a set of new workload-related features for the following five different database bindings
+
+1. MongoDB
+2. Scylla
+3. JDBC
+4. DynamoDB
+5. Couchbase3
+
+### Workload
+
+The new workload is built around a custom document structure that also comprises nested documents.
+
+```
+{
+ "airline":{
+ "name": str, pre-generated 50 unique values,
+ "alias": str, align with name
+ },
+ "src_airport": str, pre-generated 500 unique values,
+ "dst_airport": str, pre-generated 500 unique values,
+ "codeshares": str array, length 0 to 3, from airline aliases
+ "stops": int, from 0 to 3
+ "airplane": str, pre-generated 10 unique values,
+ "field1": str, random length from 0 to 1000
+}
+```
+
+The workload can be enabled by setting `workload=site.ycsb.workloads.airport.AirportWorkload`. It makes use of four different types of operations:
+
+* insert a single document
+* insert bulks of documents
+* delete a document by primary key
+* query any document by `src_airport` (equality), `dst_airport` (equality), and `stops` (lower or equal)
+* update one document by `airline.alias`
+
+### Binding
+
+In order to support this workload, the implemented bindings support two new functions compared to default YCSB: `findOne` and `updateOne`. Further, all bindings have been updated to support bulk inserts and delete by primary key.
+
+While YCSB only uses stringified content, the new workload requires the use of more diverse data types. Therefore the property`typedfields=true` needs to be set. Also, the bindings have been updated to support and distinguish between different data types. What is more, bindings for document-oriented DBMS are built such that they handle nested documents. Other bindings, in particular for JDBC and ScyllaDB are designed such that they use a flattened data model. Where possible, the updated bindings also support the definition of indexes. The full feature list is shown in the following table.
+
+| | **typed content** | **support for indexes** | **nested elements** |
+| ---------- | ----------------- | ----------------------- | ------------------- |
+| mongodb | yes | full | yes |
+| jdbc | yes | full | no |
+| couchbase3 | yes | full | yes |
+| dynamodb | yes | single column | yes |
+| scylla | yes | single column | no |
+
+The workload class can be configured to either use nested elements or use a flat data model.
+
+```
+nesteddata = false | true
+```
+
+Other changes this branch makes compared to original YCSB:
+
+* Full support for `long` (64 bits) ids so that more data can be added to the databases in the LOAD phase
+* A new Id generator that does not use a window of in-transit ids and does not crash when large amounts of data are inserted in the database. It can be enabled with `transactioninsertkeygenerator=simple`
+
+In order to support queries all bindings now come with basic support for indexes. These can be set with a binding-specific flag:
+
+```json
+couchbase.indexlist=[ ]
+dynamodb.indexlist=[ ]
+jdbc.indexlist=[ ]
+mongodb.indexlist=[ ]
+scylla.indexlist=[ ]
+```
+
+Example indexes are available in the `workloads/airport ` directory.
+
+YCSB
+====================================
+
+[](https://travis-ci.org/brianfrankcooper/YCSB)
+
+
+
+Links
+-----
+* To get here, use https://ycsb.site
+* [Our project docs](https://github.com/brianfrankcooper/YCSB/wiki)
+* [The original announcement from Yahoo!](https://labs.yahoo.com/news/yahoo-cloud-serving-benchmark/)
+
+Getting Started
+---------------
+
+1. Download the [latest release of YCSB](https://github.com/brianfrankcooper/YCSB/releases/latest):
+
+ ```sh
+ curl -O --location https://github.com/brianfrankcooper/YCSB/releases/download/0.17.0/ycsb-0.17.0.tar.gz
+ tar xfvz ycsb-0.17.0.tar.gz
+ cd ycsb-0.17.0
+ ```
+
+2. Set up a database to benchmark. There is a README file under each binding
+ directory.
+
+3. Run YCSB command.
+
+ On Linux:
+ ```sh
+ bin/ycsb.sh load basic -P workloads/workloada
+ bin/ycsb.sh run basic -P workloads/workloada
+ ```
+
+ On Windows:
+ ```bat
+ bin/ycsb.bat load basic -P workloads\workloada
+ bin/ycsb.bat run basic -P workloads\workloada
+ ```
+
+ Running the `ycsb` command without any argument will print the usage.
+
+ See https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload
+ for a detailed documentation on how to run a workload.
+
+ See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties for
+ the list of available workload properties.
+
+
+Building from source
+--------------------
+
+YCSB requires the use of Maven 3; if you use Maven 2, you may see [errors
+such as these](https://github.com/brianfrankcooper/YCSB/issues/406).
+
+To build the full distribution, with all database bindings:
+
+ mvn clean package
+
+To build a single database binding:
+
+ mvn -pl site.ycsb:mongodb-binding -am clean package
diff --git a/bin/bindings.properties b/bin/bindings.properties
new file mode 100755
index 0000000..cf1066b
--- /dev/null
+++ b/bin/bindings.properties
@@ -0,0 +1,77 @@
+#
+# Copyright (c) 2012 - 2020 YCSB contributors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+#
+
+#DATABASE BINDINGS
+#
+# Available bindings should be listed here in the form of
+# name:class
+#
+# - the name must start in column 0.
+# - the name is also the directory where the class can be found.
+# - if the directory contains multiple versions with different classes,
+# use a dash with the version. (e.g. cassandra-7, cassandra-cql)
+#
+accumulo1.9:site.ycsb.db.accumulo.AccumuloClient
+aerospike:site.ycsb.db.AerospikeClient
+asynchbase:site.ycsb.db.AsyncHBaseClient
+arangodb:site.ycsb.db.arangodb.ArangoDBClient
+arangodb3:site.ycsb.db.arangodb.ArangoDBClient
+azurecosmos:site.ycsb.db.AzureCosmosClient
+azuretablestorage:site.ycsb.db.azuretablestorage.AzureClient
+basic:site.ycsb.BasicDB
+basicts:site.ycsb.BasicTSDB
+cassandra-cql:site.ycsb.db.CassandraCQLClient
+cassandra2-cql:site.ycsb.db.CassandraCQLClient
+cloudspanner:site.ycsb.db.cloudspanner.CloudSpannerClient
+couchbase:site.ycsb.db.CouchbaseClient
+couchbase2:site.ycsb.db.couchbase2.Couchbase2Client
+couchbase3:site.ycsb.db.couchbase3.Couchbase3Client
+crail:site.ycsb.db.crail.CrailClient
+dynamodb:site.ycsb.db.DynamoDBClient
+elasticsearch:site.ycsb.db.ElasticsearchClient
+elasticsearch5:site.ycsb.db.elasticsearch5.ElasticsearchClient
+elasticsearch5-rest:site.ycsb.db.elasticsearch5.ElasticsearchRestClient
+foundationdb:site.ycsb.db.foundationdb.FoundationDBClient
+geode:site.ycsb.db.GeodeClient
+googlebigtable:site.ycsb.db.GoogleBigtableClient
+googledatastore:site.ycsb.db.GoogleDatastoreClient
+hbase1:site.ycsb.db.hbase1.HBaseClient1
+hbase2:site.ycsb.db.hbase2.HBaseClient2
+ignite:site.ycsb.db.ignite.IgniteClient
+ignite-sql:site.ycsb.db.ignite.IgniteSqlClient
+infinispan-cs:site.ycsb.db.InfinispanRemoteClient
+infinispan:site.ycsb.db.InfinispanClient
+jdbc:site.ycsb.db.JdbcDBClient
+kudu:site.ycsb.db.KuduYCSBClient
+memcached:site.ycsb.db.MemcachedClient
+mongodb:site.ycsb.db.MongoDbClient
+mongodb-async:site.ycsb.db.AsyncMongoDbClient
+nosqldb:site.ycsb.db.NoSqlDbClient
+orientdb:site.ycsb.db.OrientDBClient
+postgrenosql:site.ycsb.postgrenosql.PostgreNoSQLDBClient
+rados:site.ycsb.db.RadosClient
+redis:site.ycsb.db.RedisClient
+rest:site.ycsb.webservice.rest.RestClient
+riak:site.ycsb.db.riak.RiakKVClient
+rocksdb:site.ycsb.db.rocksdb.RocksDBClient
+s3:site.ycsb.db.S3Client
+scylla:site.ycsb.db.scylla.ScyllaCQLClient
+solr7:site.ycsb.db.solr7.SolrClient
+tarantool:site.ycsb.db.TarantoolClient
+tablestore:site.ycsb.db.tablestore.TableStoreClient
+voltdb:site.ycsb.db.voltdb.VoltClient4
+zookeeper:site.ycsb.db.zookeeper.ZKClient
\ No newline at end of file
diff --git a/bin/ycsb b/bin/ycsb
new file mode 100755
index 0000000..0104672
--- /dev/null
+++ b/bin/ycsb
@@ -0,0 +1,326 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2012 - 2020 YCSB contributors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+#
+
+import errno
+import fnmatch
+import io
+import os
+import shlex
+import sys
+import subprocess
+
+try:
+ mod = __import__('argparse')
+ import argparse
+except ImportError:
+ print >> sys.stderr, '[ERROR] argparse not found. Try installing it via "pip".'
+ exit(1)
+
+BASE_URL = "https://github.com/brianfrankcooper/YCSB/tree/master/"
+COMMANDS = {
+ "shell" : {
+ "command" : "",
+ "description" : "Interactive mode",
+ "main" : "site.ycsb.CommandLine",
+ },
+ "load" : {
+ "command" : "-load",
+ "description" : "Execute the load phase",
+ "main" : "site.ycsb.Client",
+ },
+ "run" : {
+ "command" : "-t",
+ "description" : "Execute the transaction phase",
+ "main" : "site.ycsb.Client",
+ },
+}
+
+DATABASES = {
+ "accumulo1.9" : "site.ycsb.db.accumulo.AccumuloClient",
+ "aerospike" : "site.ycsb.db.AerospikeClient",
+ "arangodb" : "site.ycsb.db.arangodb.ArangoDBClient",
+ "arangodb3" : "site.ycsb.db.arangodb.ArangoDBClient",
+ "asynchbase" : "site.ycsb.db.AsyncHBaseClient",
+ "azurecosmos" : "site.ycsb.db.AzureCosmosClient",
+ "azuretablestorage" : "site.ycsb.db.azuretablestorage.AzureClient",
+ "basic" : "site.ycsb.BasicDB",
+ "basicts" : "site.ycsb.BasicTSDB",
+ "cassandra-cql": "site.ycsb.db.CassandraCQLClient",
+ "cassandra2-cql": "site.ycsb.db.CassandraCQLClient",
+ "cloudspanner" : "site.ycsb.db.cloudspanner.CloudSpannerClient",
+ "couchbase" : "site.ycsb.db.CouchbaseClient",
+ "couchbase2" : "site.ycsb.db.couchbase2.Couchbase2Client",
+ "couchbase3" : "site.ycsb.db.couchbase3.Couchbase3Client",
+ "crail" : "site.ycsb.db.crail.CrailClient",
+ "dynamodb" : "site.ycsb.db.DynamoDBClient",
+ "elasticsearch": "site.ycsb.db.ElasticsearchClient",
+ "elasticsearch5": "site.ycsb.db.elasticsearch5.ElasticsearchClient",
+ "elasticsearch5-rest": "site.ycsb.db.elasticsearch5.ElasticsearchRestClient",
+ "foundationdb" : "site.ycsb.db.foundationdb.FoundationDBClient",
+ "geode" : "site.ycsb.db.GeodeClient",
+ "googlebigtable" : "site.ycsb.db.GoogleBigtableClient",
+ "googledatastore" : "site.ycsb.db.GoogleDatastoreClient",
+ "griddb" : "site.ycsb.db.griddb.GridDBClient",
+ "hbase1" : "site.ycsb.db.hbase1.HBaseClient1",
+ "hbase2" : "site.ycsb.db.hbase2.HBaseClient2",
+ "ignite" : "site.ycsb.db.ignite.IgniteClient",
+ "ignite-sql" : "site.ycsb.db.ignite.IgniteSqlClient",
+ "infinispan-cs": "site.ycsb.db.InfinispanRemoteClient",
+ "infinispan" : "site.ycsb.db.InfinispanClient",
+ "jdbc" : "site.ycsb.db.JdbcDBClient",
+ "kudu" : "site.ycsb.db.KuduYCSBClient",
+ "memcached" : "site.ycsb.db.MemcachedClient",
+ "maprdb" : "site.ycsb.db.mapr.MapRDBClient",
+ "maprjsondb" : "site.ycsb.db.mapr.MapRJSONDBClient",
+ "mongodb" : "site.ycsb.db.MongoDbClient",
+ "mongodb-async": "site.ycsb.db.AsyncMongoDbClient",
+ "nosqldb" : "site.ycsb.db.NoSqlDbClient",
+ "orientdb" : "site.ycsb.db.OrientDBClient",
+ "postgrenosql" : "site.ycsb.postgrenosql.PostgreNoSQLDBClient",
+ "rados" : "site.ycsb.db.RadosClient",
+ "redis" : "site.ycsb.db.RedisClient",
+ "rest" : "site.ycsb.webservice.rest.RestClient",
+ "riak" : "site.ycsb.db.riak.RiakKVClient",
+ "rocksdb" : "site.ycsb.db.rocksdb.RocksDBClient",
+ "s3" : "site.ycsb.db.S3Client",
+ "seaweedfs" : "site.ycsb.db.seaweed.SeaweedClient",
+ "scylla" : "site.ycsb.db.scylla.ScyllaCQLClient",
+ "solr7" : "site.ycsb.db.solr7.SolrClient",
+ "tarantool" : "site.ycsb.db.TarantoolClient",
+ "tablestore" : "site.ycsb.db.tablestore.TableStoreClient",
+ "zookeeper" : "site.ycsb.db.zookeeper.ZKClient"
+}
+
+OPTIONS = {
+ "-P file" : "Specify workload file",
+ "-p key=value" : "Override workload property",
+ "-s" : "Print status to stderr",
+ "-target n" : "Target ops/sec (default: unthrottled)",
+ "-threads n" : "Number of client threads (default: 1)",
+ "-cp path" : "Additional Java classpath entries",
+ "-jvm-args args" : "Additional arguments to the JVM",
+}
+
+def usage():
+ output = io.BytesIO()
+ print >> output, "%s command database [options]" % sys.argv[0]
+
+ print >> output, "\nCommands:"
+ for command in sorted(COMMANDS.keys()):
+ print >> output, " %s %s" % (command.ljust(14),
+ COMMANDS[command]["description"])
+
+ print >> output, "\nDatabases:"
+ for db in sorted(DATABASES.keys()):
+ print >> output, " %s %s" % (db.ljust(14), BASE_URL +
+ db.split("-")[0])
+
+ print >> output, "\nOptions:"
+ for option in sorted(OPTIONS.keys()):
+ print >> output, " %s %s" % (option.ljust(14), OPTIONS[option])
+
+ print >> output, """\nWorkload Files:
+ There are various predefined workloads under workloads/ directory.
+ See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties
+ for the list of workload properties."""
+
+ return output.getvalue()
+
+# Python 2.6 doesn't have check_output. Add the method as it is in Python 2.7
+# Based on https://github.com/python/cpython/blob/2.7/Lib/subprocess.py#L545
+def check_output(*popenargs, **kwargs):
+ r"""Run command with arguments and return its output as a byte string.
+
+ If the exit code was non-zero it raises a CalledProcessError. The
+ CalledProcessError object will have the return code in the returncode
+ attribute and output in the output attribute.
+
+ The arguments are the same as for the Popen constructor. Example:
+
+ >>> check_output(["ls", "-l", "/dev/null"])
+ 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
+
+ The stdout argument is not allowed as it is used internally.
+ To capture standard error in the result, use stderr=STDOUT.
+
+ >>> check_output(["/bin/sh", "-c",
+ ... "ls -l non_existent_file ; exit 0"],
+ ... stderr=STDOUT)
+ 'ls: non_existent_file: No such file or directory\n'
+ """
+ if 'stdout' in kwargs:
+ raise ValueError('stdout argument not allowed, it will be overridden.')
+ process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+ output, unused_err = process.communicate()
+ retcode = process.poll()
+ if retcode:
+ cmd = kwargs.get("args")
+ if cmd is None:
+ cmd = popenargs[0]
+ error = subprocess.CalledProcessError(retcode, cmd)
+ error.output = output
+ raise error
+ return output
+
+def debug(message):
+ print >> sys.stderr, "[DEBUG] ", message
+
+def warn(message):
+ print >> sys.stderr, "[WARN] ", message
+
+def error(message):
+ print >> sys.stderr, "[ERROR] ", message
+
+def find_jars(dir, glob='*.jar'):
+ jars = []
+ for (dirpath, dirnames, filenames) in os.walk(dir):
+ for filename in fnmatch.filter(filenames, glob):
+ jars.append(os.path.join(dirpath, filename))
+ return jars
+
+def get_ycsb_home():
+ dir = os.path.abspath(os.path.dirname(sys.argv[0]))
+ while "LICENSE.txt" not in os.listdir(dir):
+ dir = os.path.join(dir, os.path.pardir)
+ return os.path.abspath(dir)
+
+def is_distribution():
+ # If there's a top level pom, we're a source checkout. otherwise a dist artifact
+ return "pom.xml" not in os.listdir(get_ycsb_home())
+
+# Run the maven dependency plugin to get the local jar paths.
+# presumes maven can run, so should only be run on source checkouts
+# will invoke the 'package' goal for the given binding in order to resolve intra-project deps
+# presumes maven properly handles system-specific path separators
+# Given module is full module name eg. 'core' or 'couchbase-binding'
+def get_classpath_from_maven(module):
+ try:
+ debug("Running 'mvn -pl site.ycsb:" + module + " -am package -DskipTests "
+ "dependency:build-classpath -DincludeScope=compile -Dmdep.outputFilterFile=true'")
+ mvn_output = check_output(["mvn", "-pl", "site.ycsb:" + module,
+ "-am", "package", "-DskipTests",
+ "dependency:build-classpath",
+ "-DincludeScope=compile",
+ "-Dmdep.outputFilterFile=true"])
+ # the above outputs a "classpath=/path/tojar:/path/to/other/jar" for each module
+ # the last module will be the datastore binding
+ line = [x for x in mvn_output.splitlines() if x.startswith("classpath=")][-1:]
+ return line[0][len("classpath="):]
+ except subprocess.CalledProcessError, err:
+ error("Attempting to generate a classpath from Maven failed "
+ "with return code '" + str(err.returncode) + "'. The output from "
+ "Maven follows, try running "
+ "'mvn -DskipTests package dependency:build=classpath' on your "
+ "own and correct errors." + os.linesep + os.linesep + "mvn output:" + os.linesep
+ + err.output)
+ sys.exit(err.returncode)
+
+def main():
+ p = argparse.ArgumentParser(
+ usage=usage(),
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ p.add_argument('-cp', dest='classpath', help="""Additional classpath
+ entries, e.g. '-cp /tmp/hbase-1.0.1.1/conf'. Will be
+ prepended to the YCSB classpath.""")
+ p.add_argument("-jvm-args", default=[], type=shlex.split,
+ help="""Additional arguments to pass to 'java', e.g.
+ '-Xmx4g'""")
+ p.add_argument("command", choices=sorted(COMMANDS),
+ help="""Command to run.""")
+ p.add_argument("database", choices=sorted(DATABASES),
+ help="""Database to test.""")
+ args, remaining = p.parse_known_args()
+ ycsb_home = get_ycsb_home()
+
+ # Use JAVA_HOME to find java binary if set, otherwise just use PATH.
+ java = "java"
+ java_home = os.getenv("JAVA_HOME")
+ if java_home:
+ java = os.path.join(java_home, "bin", "java")
+ db_classname = DATABASES[args.database]
+ command = COMMANDS[args.command]["command"]
+ main_classname = COMMANDS[args.command]["main"]
+
+ # Classpath set up
+ binding = args.database.split("-")[0]
+
+ if binding == "cassandra2":
+ warn("The 'cassandra2-cql' client has been deprecated. It has been "
+ "renamed to simply 'cassandra-cql'. This alias will be removed"
+ " in the next YCSB release.")
+ binding = "cassandra"
+
+ if binding == "couchbase":
+ warn("The 'couchbase' client has been deprecated. If you are using "
+ "Couchbase 4.0+ try using the 'couchbase2' client instead.")
+
+ if binding == "hbase14":
+ warn("The 'hbase14' client has been deprecated. HBase 1.y users should "
+ "rely on the 'hbase1' client instead.")
+ binding = "hbase1"
+
+ if binding == "arangodb3":
+ warn("The 'arangodb3' client has been deprecated. The binding 'arangodb' "
+ "now covers every ArangoDB version. This alias will be removed "
+ "in the next YCSB release.")
+ binding = "arangodb"
+
+ if is_distribution():
+ db_dir = os.path.join(ycsb_home, binding + "-binding")
+ # include top-level conf for when we're a binding-specific artifact.
+ # If we add top-level conf to the general artifact, starting here
+ # will allow binding-specific conf to override (because it's prepended)
+ cp = [os.path.join(ycsb_home, "conf")]
+ cp.extend(find_jars(os.path.join(ycsb_home, "lib")))
+ cp.extend(find_jars(os.path.join(db_dir, "lib")))
+ else:
+ warn("Running against a source checkout. In order to get our runtime "
+ "dependencies we'll have to invoke Maven. Depending on the state "
+ "of your system, this may take ~30-45 seconds")
+ db_location = "core" if (binding == "basic" or binding == "basicts") else binding
+ project = "core" if (binding == "basic" or binding == "basicts") else binding + "-binding"
+ db_dir = os.path.join(ycsb_home, db_location)
+ # goes first so we can rely on side-effect of package
+ maven_says = get_classpath_from_maven(project)
+ # TODO when we have a version property, skip the glob
+ cp = find_jars(os.path.join(db_dir, "target"),
+ project + "*.jar")
+ # alredy in jar:jar:jar form
+ cp.append(maven_says)
+ cp.insert(0, os.path.join(db_dir, "conf"))
+ classpath = os.pathsep.join(cp)
+ if args.classpath:
+ classpath = os.pathsep.join([args.classpath, classpath])
+
+ ycsb_command = ([java] + args.jvm_args +
+ ["-cp", classpath,
+ main_classname, "-db", db_classname] + remaining)
+ if command:
+ ycsb_command.append(command)
+ print >> sys.stderr, " ".join(ycsb_command)
+ try:
+ return subprocess.call(ycsb_command)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ error('Command failed. Is java installed and on your PATH?')
+ return 1
+ else:
+ raise
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/bin/ycsb.bat b/bin/ycsb.bat
new file mode 100755
index 0000000..7802659
--- /dev/null
+++ b/bin/ycsb.bat
@@ -0,0 +1,231 @@
+@REM
+@REM Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved.
+@REM
+@REM Licensed under the Apache License, Version 2.0 (the "License"); you
+@REM may not use this file except in compliance with the License. You
+@REM may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing, software
+@REM distributed under the License is distributed on an "AS IS" BASIS,
+@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+@REM implied. See the License for the specific language governing
+@REM permissions and limitations under the License. See accompanying
+@REM LICENSE file.
+@REM
+@REM -----------------------------------------------------------------------
+@REM Control Script for YCSB
+@REM
+@REM Environment Variable Prerequisites
+@REM
+@REM Do not set the variables in this script. Instead put them into a script
+@REM setenv.sh in YCSB_HOME/bin to keep your customizations separate.
+@REM
+@REM YCSB_HOME (Optional) YCSB installation directory. If not set
+@REM this script will use the parent directory of where this
+@REM script is run from.
+@REM
+@REM JAVA_HOME (Required) Must point at your Java Development Kit
+@REM or Java Runtime Environment installation.
+@REM
+@REM JAVA_OPTS (Optional) Java runtime options used when any command
+@REM is executed.
+@REM
+@REM WARNING!!! YCSB home must be located in a directory path that doesn't
+@REM contain spaces.
+@REM
+
+@ECHO OFF
+SETLOCAL ENABLEDELAYEDEXPANSION
+
+@REM Only set YCSB_HOME if not already set
+PUSHD %~dp0..
+IF NOT DEFINED YCSB_HOME SET YCSB_HOME=%CD%
+POPD
+
+@REM Ensure that any extra CLASSPATH variables are set via setenv.bat
+SET CLASSPATH=
+
+@REM Pull in customization options
+if exist "%YCSB_HOME%\bin\setenv.bat" call "%YCSB_HOME%\bin\setenv.bat"
+
+@REM Check if we have a usable JDK
+IF "%JAVA_HOME%." == "." GOTO noJavaHome
+IF NOT EXIST "%JAVA_HOME%\bin\java.exe" GOTO noJavaHome
+GOTO okJava
+:noJavaHome
+ECHO The JAVA_HOME environment variable is not defined correctly.
+GOTO exit
+:okJava
+
+@REM Determine YCSB command argument
+IF NOT "load" == "%1" GOTO noload
+SET YCSB_COMMAND=-load
+SET YCSB_CLASS=site.ycsb.Client
+GOTO gotCommand
+:noload
+IF NOT "run" == "%1" GOTO noRun
+SET YCSB_COMMAND=-t
+SET YCSB_CLASS=site.ycsb.Client
+GOTO gotCommand
+:noRun
+IF NOT "shell" == "%1" GOTO noShell
+SET YCSB_COMMAND=
+SET YCSB_CLASS=site.ycsb.CommandLine
+GOTO gotCommand
+:noShell
+ECHO [ERROR] Found unknown command '%1'
+ECHO [ERROR] Expected one of 'load', 'run', or 'shell'. Exiting.
+GOTO exit
+:gotCommand
+
+@REM Find binding information
+FOR /F "delims=" %%G in (
+ 'FINDSTR /B "%2:" %YCSB_HOME%\bin\bindings.properties'
+) DO SET "BINDING_LINE=%%G"
+
+IF NOT "%BINDING_LINE%." == "." GOTO gotBindingLine
+ECHO [ERROR] The specified binding '%2' was not found. Exiting.
+GOTO exit
+:gotBindingLine
+
+@REM Pull out binding name and class
+FOR /F "tokens=1-2 delims=:" %%G IN ("%BINDING_LINE%") DO (
+ SET BINDING_NAME=%%G
+ SET BINDING_CLASS=%%H
+)
+
+@REM Some bindings have multiple versions that are managed in the same
+@REM directory.
+@REM They are noted with a '-' after the binding name.
+@REM (e.g. cassandra-7 & cassandra-8)
+FOR /F "tokens=1 delims=-" %%G IN ("%BINDING_NAME%") DO (
+ SET BINDING_DIR=%%G
+)
+
+@REM The 'basic' binding is core functionality
+IF NOT "%BINDING_NAME%" == "basic" GOTO noBasic
+SET BINDING_DIR=core
+:noBasic
+
+@REM Add Top level conf to classpath
+IF "%CLASSPATH%." == "." GOTO emptyClasspath
+SET CLASSPATH=%CLASSPATH%;%YCSB_HOME%\conf
+GOTO confAdded
+:emptyClasspath
+SET CLASSPATH=%YCSB_HOME%\conf
+:confAdded
+
+@REM Cassandra2 deprecation message
+IF NOT "%BINDING_DIR%" == "cassandra2" GOTO notAliasCassandra
+echo [WARN] The 'cassandra2-cql' client has been deprecated. It has been renamed to simply 'cassandra-cql'. This alias will be removed in the next YCSB release.
+SET BINDING_DIR=cassandra
+:notAliasCassandra
+
+@REM hbase14 replaced with hbase1
+IF NOT "%BINDING_DIR%" == "hbase14" GOTO notAliasHBase14
+echo [WARN] The 'hbase14' client has been deprecated. HBase 1.y users should rely on the 'hbase1' client instead.
+SET BINDING_DIR=hbase1
+:notAliasHBase14
+
+@REM arangodb3 deprecation message
+IF NOT "%BINDING_DIR%" == "arangodb3" GOTO notAliasArangodb3
+echo [WARN] The 'arangodb3' client has been deprecated. The binding 'arangodb' now covers every ArangoDB version. This alias will be removed in the next YCSB release.
+SET BINDING_DIR=arangodb
+:notAliasArangodb3
+
+@REM Build classpath according to source checkout or release distribution
+IF EXIST "%YCSB_HOME%\pom.xml" GOTO gotSource
+@REM Build classpath according to source checkout or release distribution
+IF EXIST "%YCSB_HOME%\pom.xml" GOTO gotSource
+
+@REM Core libraries
+FOR %%F IN (%YCSB_HOME%\lib\*.jar) DO (
+ SET CLASSPATH=!CLASSPATH!;%%F%
+)
+
+@REM Database conf dir
+IF NOT EXIST "%YCSB_HOME%\%BINDING_DIR%-binding\conf" GOTO noBindingConf
+set CLASSPATH=%CLASSPATH%;%YCSB_HOME%\%BINDING_DIR%-binding\conf
+:noBindingConf
+
+@REM Database libraries
+FOR %%F IN (%YCSB_HOME%\%BINDING_DIR%-binding\lib\*.jar) DO (
+ SET CLASSPATH=!CLASSPATH!;%%F%
+)
+GOTO classpathComplete
+
+:gotSource
+@REM Check for some basic libraries to see if the source has been built.
+IF EXIST "%YCSB_HOME%\core\target\dependency\*.jar" (
+ IF EXIST "%YCSB_HOME%\%BINDING_DIR%\target\*.jar" (
+ GOTO gotJars
+ )
+)
+
+@REM Call mvn to build source checkout.
+IF "%BINDING_NAME%" == "basic" GOTO buildCore
+SET MVN_PROJECT=%BINDING_DIR%-binding
+goto gotMvnProject
+:buildCore
+SET MVN_PROJECT=core
+:gotMvnProject
+
+ECHO [WARN] YCSB libraries not found. Attempting to build...
+CALL mvn -Psource-run -pl site.ycsb:%MVN_PROJECT% -am package -DskipTests
+IF %ERRORLEVEL% NEQ 0 (
+ ECHO [ERROR] Error trying to build project. Exiting.
+ GOTO exit
+)
+
+:gotJars
+@REM Core libraries
+FOR %%F IN (%YCSB_HOME%\core\target\*.jar) DO (
+ SET CLASSPATH=!CLASSPATH!;%%F%
+)
+
+@REM Core dependency libraries
+FOR %%F IN (%YCSB_HOME%\core\target\dependency\*.jar) DO (
+ SET CLASSPATH=!CLASSPATH!;%%F%
+)
+
+@REM Database conf (need to find because location is not consistent)
+FOR /D /R %YCSB_HOME%\%BINDING_DIR% %%F IN (*) DO (
+ IF "%%~nxF" == "conf" SET CLASSPATH=!CLASSPATH!;%%F%
+)
+
+@REM Database libraries
+FOR %%F IN (%YCSB_HOME%\%BINDING_DIR%\target\*.jar) DO (
+ SET CLASSPATH=!CLASSPATH!;%%F%
+)
+
+@REM Database dependency libraries
+FOR %%F IN (%YCSB_HOME%\%BINDING_DIR%\target\dependency\*.jar) DO (
+ SET CLASSPATH=!CLASSPATH!;%%F%
+)
+
+:classpathComplete
+
+@REM Couchbase deprecation message
+IF NOT "%BINDING_DIR%" == "couchbase" GOTO notOldCouchbase
+echo [WARN] The 'couchbase' client is deprecated. If you are using Couchbase 4.0+ try using the 'couchbase2' client instead.
+:notOldCouchbase
+
+@REM Get the rest of the arguments, skipping the first 2
+FOR /F "tokens=2*" %%G IN ("%*") DO (
+ SET YCSB_ARGS=%%H
+)
+
+@REM Run YCSB
+@ECHO ON
+"%JAVA_HOME%\bin\java.exe" %JAVA_OPTS% -classpath "%CLASSPATH%" %YCSB_CLASS% %YCSB_COMMAND% -db %BINDING_CLASS% %YCSB_ARGS%
+@ECHO OFF
+
+GOTO end
+
+:exit
+EXIT /B 1;
+
+:end
+
diff --git a/bin/ycsb.sh b/bin/ycsb.sh
new file mode 100755
index 0000000..7a1165b
--- /dev/null
+++ b/bin/ycsb.sh
@@ -0,0 +1,260 @@
+#!/bin/sh
+#
+# Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+#
+# -----------------------------------------------------------------------------
+# Control Script for YCSB
+#
+# Environment Variable Prerequisites
+#
+# Do not set the variables in this script. Instead put them into a script
+# setenv.sh in YCSB_HOME/bin to keep your customizations separate.
+#
+# YCSB_HOME (Optional) YCSB installation directory. If not set
+# this script will use the parent directory of where this
+# script is run from.
+#
+# JAVA_HOME (Optional) Must point at your Java Development Kit
+# installation. If empty, this script tries use the
+# available java executable.
+#
+# JAVA_OPTS (Optional) Java runtime options used when any command
+# is executed.
+#
+# WARNING!!! YCSB home must be located in a directory path that doesn't
+# contain spaces.
+#
+# www.shellcheck.net was used to validate this script
+
+# Cygwin support
+CYGWIN=false
+case "$(uname)" in
+CYGWIN*) CYGWIN=true;;
+esac
+
+# Get script path
+SCRIPT_DIR=$(dirname "$0" 2>/dev/null)
+
+# Only set YCSB_HOME if not already set
+[ -z "$YCSB_HOME" ] && YCSB_HOME=$(cd "$SCRIPT_DIR/.." || exit; pwd)
+
+# Ensure that any extra CLASSPATH variables are set via setenv.sh
+CLASSPATH=
+
+# Pull in customization options
+if [ -r "$YCSB_HOME/bin/setenv.sh" ]; then
+ # Shellcheck wants a source, but this directive only runs if available
+ # So, tell shellcheck to ignore
+ # shellcheck source=/dev/null
+ . "$YCSB_HOME/bin/setenv.sh"
+fi
+
+# Attempt to find the available JAVA, if JAVA_HOME not set
+if [ -z "$JAVA_HOME" ]; then
+ JAVA_PATH=$(which java 2>/dev/null)
+ if [ "x$JAVA_PATH" != "x" ]; then
+ JAVA_HOME=$(dirname "$(dirname "$JAVA_PATH" 2>/dev/null)")
+ fi
+fi
+
+# If JAVA_HOME still not set, error
+if [ -z "$JAVA_HOME" ]; then
+ echo "[ERROR] Java executable not found. Exiting."
+ exit 1;
+fi
+
+# Determine YCSB command argument
+if [ "load" = "$1" ] ; then
+ YCSB_COMMAND=-load
+ YCSB_CLASS=site.ycsb.Client
+elif [ "run" = "$1" ] ; then
+ YCSB_COMMAND=-t
+ YCSB_CLASS=site.ycsb.Client
+elif [ "shell" = "$1" ] ; then
+ YCSB_COMMAND=
+ YCSB_CLASS=site.ycsb.CommandLine
+else
+ echo "[ERROR] Found unknown command '$1'"
+ echo "[ERROR] Expected one of 'load', 'run', or 'shell'. Exiting."
+ exit 1;
+fi
+
+# Find binding information
+BINDING_LINE=$(grep "^$2:" "$YCSB_HOME/bin/bindings.properties" -m 1)
+
+if [ -z "$BINDING_LINE" ] ; then
+ echo "[ERROR] The specified binding '$2' was not found. Exiting."
+ exit 1;
+fi
+
+# Get binding name and class
+BINDING_NAME=$(echo "$BINDING_LINE" | cut -d':' -f1)
+BINDING_CLASS=$(echo "$BINDING_LINE" | cut -d':' -f2)
+
+# Some bindings have multiple versions that are managed in the same directory.
+# They are noted with a '-' after the binding name.
+# (e.g. cassandra-7 & cassandra-8)
+BINDING_DIR=$(echo "$BINDING_NAME" | cut -d'-' -f1)
+
+# The 'basic' binding is core functionality
+if [ "$BINDING_NAME" = "basic" ] ; then
+ BINDING_DIR=core
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $CYGWIN; then
+ [ -n "$JAVA_HOME" ] && JAVA_HOME=$(cygpath --unix "$JAVA_HOME")
+ [ -n "$CLASSPATH" ] && CLASSPATH=$(cygpath --path --unix "$CLASSPATH")
+fi
+
+# Check if source checkout, or release distribution
+DISTRIBUTION=true
+if [ -r "$YCSB_HOME/pom.xml" ]; then
+ DISTRIBUTION=false;
+fi
+
+# Add Top level conf to classpath
+if [ -z "$CLASSPATH" ] ; then
+ CLASSPATH="$YCSB_HOME/conf"
+else
+ CLASSPATH="$CLASSPATH:$YCSB_HOME/conf"
+fi
+
+# Cassandra2 deprecation message
+if [ "${BINDING_DIR}" = "cassandra2" ] ; then
+ echo "[WARN] The 'cassandra2-cql' client has been deprecated. It has been \
+renamed to simply 'cassandra-cql'. This alias will be removed in the next \
+YCSB release."
+ BINDING_DIR="cassandra"
+fi
+
+# hbase14 replaced by hbas1
+if [ "${BINDING_DIR}" = "hbase14" ] ; then
+ echo "[WARN] The 'hbase14' client has been deprecated. HBase 1.y users should \
+rely on the 'hbase1' client instead."
+ BINDING_DIR="hbase1"
+fi
+
+# arangodb3 deprecation message
+if [ "${BINDING_DIR}" = "arangodb3" ] ; then
+ echo "[WARN] The 'arangodb3' client has been deprecated. The binding 'arangodb' \
+now covers every ArangoDB version. This alias will be removed \
+in the next YCSB release."
+ BINDING_DIR="arangodb"
+fi
+
+# Build classpath
+# The "if" check after the "for" is because glob may just return the pattern
+# when no files are found. The "if" makes sure the file is really there.
+if $DISTRIBUTION; then
+ # Core libraries
+ for f in "$YCSB_HOME"/lib/*.jar ; do
+ if [ -r "$f" ] ; then
+ CLASSPATH="$CLASSPATH:$f"
+ fi
+ done
+
+ # Database conf dir
+ if [ -r "$YCSB_HOME"/"$BINDING_DIR"-binding/conf ] ; then
+ CLASSPATH="$CLASSPATH:$YCSB_HOME/$BINDING_DIR-binding/conf"
+ fi
+
+ # Database libraries
+ for f in "$YCSB_HOME"/"$BINDING_DIR"-binding/lib/*.jar ; do
+ if [ -r "$f" ] ; then
+ CLASSPATH="$CLASSPATH:$f"
+ fi
+ done
+
+# Source checkout
+else
+ # Check for some basic libraries to see if the source has been built.
+ if ! ls "$YCSB_HOME"/core/target/*.jar 1> /dev/null 2>&1 || \
+ ! ls "$YCSB_HOME"/"$BINDING_DIR"/target/*.jar 1>/dev/null 2>&1; then
+ # Call mvn to build source checkout.
+ if [ "$BINDING_NAME" = "basic" ] ; then
+ MVN_PROJECT=core
+ else
+ MVN_PROJECT="$BINDING_DIR"-binding
+ fi
+
+ echo "[WARN] YCSB libraries not found. Attempting to build..."
+ if mvn -Psource-run -pl site.ycsb:"$MVN_PROJECT" -am package -DskipTests; then
+ echo "[ERROR] Error trying to build project. Exiting."
+ exit 1;
+ fi
+ fi
+
+ # Core libraries
+ for f in "$YCSB_HOME"/core/target/*.jar ; do
+ if [ -r "$f" ] ; then
+ CLASSPATH="$CLASSPATH:$f"
+ fi
+ done
+
+ # Core dependency libraries
+ for f in "$YCSB_HOME"/core/target/dependency/*.jar ; do
+ if [ -r "$f" ] ; then
+ CLASSPATH="$CLASSPATH:$f"
+ fi
+ done
+
+ # Database conf (need to find because location is not consistent)
+ CLASSPATH_CONF=$(find "$YCSB_HOME"/$BINDING_DIR -name "conf" | while IFS="" read -r file; do echo ":$file"; done)
+ if [ "x$CLASSPATH_CONF" != "x" ]; then
+ CLASSPATH="$CLASSPATH$CLASSPATH_CONF"
+ fi
+
+ # Database libraries
+ for f in "$YCSB_HOME"/"$BINDING_DIR"/target/*.jar ; do
+ if [ -r "$f" ] ; then
+ CLASSPATH="$CLASSPATH:$f"
+ fi
+ done
+
+ # Database dependency libraries
+ for f in "$YCSB_HOME"/"$BINDING_DIR"/target/dependency/*.jar ; do
+ if [ -r "$f" ] ; then
+ CLASSPATH="$CLASSPATH:$f"
+ fi
+ done
+fi
+
+# Couchbase deprecation message
+if [ "${BINDING_DIR}" = "couchbase" ] ; then
+ echo "[WARN] The 'couchbase' client is deprecated. If you are using \
+Couchbase 4.0+ try using the 'couchbase2' client instead."
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $CYGWIN; then
+ [ -n "$JAVA_HOME" ] && JAVA_HOME=$(cygpath --unix "$JAVA_HOME")
+ [ -n "$CLASSPATH" ] && CLASSPATH=$(cygpath --path --windows "$CLASSPATH")
+fi
+
+# Get the rest of the arguments
+YCSB_ARGS=$(echo "$@" | cut -d' ' -f3-)
+
+# About to run YCSB
+echo "$JAVA_HOME/bin/java $JAVA_OPTS -classpath $CLASSPATH $YCSB_CLASS $YCSB_COMMAND -db $BINDING_CLASS $YCSB_ARGS"
+
+# Run YCSB
+# Shellcheck reports the following line as needing double quotes to prevent
+# globbing and word splitting. However, word splitting is the desired effect
+# here. So, the shellcheck error is disabled for this line.
+# shellcheck disable=SC2086
+"$JAVA_HOME/bin/java" $JAVA_OPTS -classpath "$CLASSPATH" $YCSB_CLASS $YCSB_COMMAND -db $BINDING_CLASS $YCSB_ARGS
+
diff --git a/binding-parent/datastore-specific-descriptor/pom.xml b/binding-parent/datastore-specific-descriptor/pom.xml
new file mode 100644
index 0000000..afcb1fc
--- /dev/null
+++ b/binding-parent/datastore-specific-descriptor/pom.xml
@@ -0,0 +1,44 @@
+
+
+
+ 4.0.0
+
+
+ site.ycsb
+ root
+ 0.18.0-SNAPSHOT
+ ../../
+
+
+ datastore-specific-descriptor
+ Per Datastore Binding descriptor
+ jar
+
+
+ This module contains the assembly descriptor used by the individual components
+ to build binding-specific distributions.
+
+
+
+ site.ycsb
+ core
+ ${project.version}
+
+
+
+
diff --git a/binding-parent/datastore-specific-descriptor/src/main/resources/assemblies/datastore-specific-assembly.xml b/binding-parent/datastore-specific-descriptor/src/main/resources/assemblies/datastore-specific-assembly.xml
new file mode 100644
index 0000000..05273f8
--- /dev/null
+++ b/binding-parent/datastore-specific-descriptor/src/main/resources/assemblies/datastore-specific-assembly.xml
@@ -0,0 +1,85 @@
+
+
+
+ dist
+ true
+ ycsb-${artifactId}-${version}
+
+
+ README.md
+
+
+
+
+
+ ..
+
+ 0644
+
+ LICENSE.txt
+ NOTICE.txt
+
+
+
+ ../bin
+ bin
+ 0755
+
+ ycsb*
+
+
+
+ ../bin
+ bin
+ 0644
+
+ bindings.properties
+
+
+
+ ../workloads
+ workloads
+ 0644
+
+
+ src/main/conf
+ conf
+ 0644
+
+
+
+
+ lib
+
+ site.ycsb:core
+
+ provided
+ true
+
+
+ lib
+
+ *:jar:*
+
+
+ *:sources
+
+
+
+
diff --git a/binding-parent/pom.xml b/binding-parent/pom.xml
new file mode 100644
index 0000000..77cc2a4
--- /dev/null
+++ b/binding-parent/pom.xml
@@ -0,0 +1,205 @@
+
+
+
+ 4.0.0
+
+
+ site.ycsb
+ root
+ 0.18.0-SNAPSHOT
+
+
+ binding-parent
+ YCSB Datastore Binding Parent
+ pom
+
+
+ This module acts as the parent for new datastore bindings.
+ It creates a datastore specific binary artifact.
+
+
+
+ datastore-specific-descriptor
+
+
+
+
+ false
+ false
+ false
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+ ${maven.assembly.version}
+
+
+ site.ycsb
+ datastore-specific-descriptor
+ ${project.version}
+
+
+
+
+ datastore-specific-assembly
+
+ ycsb-${project.artifactId}-${project.version}
+
+ tar.gz
+
+ false
+ posix
+
+
+
+ package
+
+ single
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-checkstyle-plugin
+
+
+ validate
+
+ ../checkstyle.xml
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+ ${maven.dependency.version}
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+
+ stage-dependencies
+ package
+
+ copy-dependencies
+
+
+ runtime
+
+
+
+
+
+
+
+
+
+ datastore-binding
+
+
+ README.md
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+
+
+
+
+
+
+ tests-on-jdk9
+
+ 9
+
+
+ ${skipJDK9Tests}
+
+
+
+
+ tests-on-jdk10
+
+ 10
+
+
+ ${skipJDK10Tests}
+
+
+
+
+ tests-on-jdk11
+
+ 11
+
+
+ ${skipJDK11Tests}
+
+
+
+
+ ycsb-release
+
+
+ true
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+ but-still-deploy-the-binding-parent
+
+ deploy
+
+ deploy
+ false
+
+ false
+
+
+
+
+
+
+
+
+
+
diff --git a/checkstyle.xml b/checkstyle.xml
new file mode 100644
index 0000000..de1166c
--- /dev/null
+++ b/checkstyle.xml
@@ -0,0 +1,193 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/core/CHANGES.md b/core/CHANGES.md
new file mode 100644
index 0000000..8b651bc
--- /dev/null
+++ b/core/CHANGES.md
@@ -0,0 +1,84 @@
+
+
+When used as a latency under load benchmark YCSB in it's original form suffers from
+Coordinated Omission[1] and related measurement issue:
+
+* Load is controlled by response time
+* Measurement does not account for missing time
+* Measurement starts at beginning of request rather than at intended beginning
+* Measurement is limited in scope as the histogram does not provide data on overflow values
+
+To provide a minimal correction patch the following were implemented:
+
+1. Replace internal histogram implementation with HdrHistogram[2]:
+HdrHistogram offers a dynamic range of measurement at a given precision and will
+improve the fidelity of reporting. It allows capturing a much wider range of latencies.
+HdrHistogram also supports compressed loss-less serialization which enable capturing
+snapshot histograms from which lower resolution histograms can be constructed for plotting
+latency over time. Snapshot interval histograms are serialized on status reporting which
+must be enabled using the '-s' option.
+
+2. Track intended operation start and report latencies from that point in time:
+Assuming the benchmark sets a target schedule of execution in which every operation
+is supposed to happen at a given time the benchmark should measure the latency between
+intended start time and operation completion.
+This required the introduction of a new measurement point and inevitably
+includes measuring some of the internal preparation steps of the load generator.
+These overhead should be negligible in the context of a network hop, but could
+be corrected for by estimating the load-generator overheads (e.g. by measuring a
+no-op DB or by measuring the setup time for an operation and deducting that from total).
+This intended measurement point is only used when there is a target load (specified by
+the -target paramaeter)
+
+This branch supports the following new options:
+
+* -p measurementtype=[histogram|hdrhistogram|hdrhistogram+histogram|timeseries] (default=histogram)
+The new measurement types are hdrhistogram and hdrhistogram+histogram. Default is still
+histogram, which is the old histogram. Ultimately we would remove the old measurement types
+and use only HdrHistogram but the old measurement is left in there for comparison sake.
+
+* -p measurement.interval=[op|intended|both] (default=op)
+This new option deferentiates between measured intervals and adds the intended interval(as described)
+above, and the option to record both the op and intended for comparison.
+
+* -p hdrhistogram.fileoutput=[true|false] (default=false)
+This new option will enable periodical writes of the interval histogram into an output file. The path can be set using '-p hdrhistogram.output.path='.
+
+Example parameters:
+-target 1000 -s -p workload=site.ycsb.workloads.CoreWorkload -p basicdb.verbose=false -p basicdb.simulatedelay=4 -p measurement.interval=both -p measurementtype=hdrhistogram -p hdrhistogram.fileoutput=true -p maxexecutiontime=60
+
+Further changes made:
+
+* -p status.interval= (default=10)
+Controls the number of seconds between status reports and therefore between HdrHistogram snapshots reported.
+
+* -p basicdb.randomizedelay=[true|false] (default=true)
+Controls weather the delay simulated by the mock DB is uniformly random or not.
+
+Further suggestions:
+
+1. Correction load control: currently after a pause the load generator will do
+operations back to back to catchup, this leads to a flat out throughput mode
+of testing as opposed to controlled load.
+
+2. Move to async model: Scenarios where Ops have no dependency could delegate the
+Op execution to a threadpool and thus separate the request rate control from the
+synchronous execution of Ops. Measurement would start on queuing for execution.
+
+1. https://groups.google.com/forum/#!msg/mechanical-sympathy/icNZJejUHfE/BfDekfBEs_sJ
+2. https://github.com/HdrHistogram/HdrHistogram
\ No newline at end of file
diff --git a/core/pom.xml b/core/pom.xml
new file mode 100644
index 0000000..7013b0c
--- /dev/null
+++ b/core/pom.xml
@@ -0,0 +1,99 @@
+
+
+
+
+ 4.0.0
+
+ site.ycsb
+ root
+ 0.18.0-SNAPSHOT
+
+
+ core
+ Core YCSB
+ jar
+
+
+ 1.9.4
+
+
+
+
+ org.apache.htrace
+ htrace-core4
+ 4.1.0-incubating
+
+
+ org.codehaus.jackson
+ jackson-mapper-asl
+ ${jackson.api.version}
+
+
+ org.codehaus.jackson
+ jackson-core-asl
+ ${jackson.api.version}
+
+
+ org.testng
+ testng
+ 6.1.1
+ test
+
+
+ org.hdrhistogram
+ HdrHistogram
+ 2.1.4
+
+
+
+
+
+
+ src/main/resources
+ true
+
+
+
+
+
+
+
+ source-run
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+
+ stage-dependencies
+ package
+
+ copy-dependencies
+
+
+ runtime
+
+
+
+
+
+
+
+
+
diff --git a/core/src/main/java/site/ycsb/BasicDB.java b/core/src/main/java/site/ycsb/BasicDB.java
new file mode 100644
index 0000000..1b66a3f
--- /dev/null
+++ b/core/src/main/java/site/ycsb/BasicDB.java
@@ -0,0 +1,484 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ * Copyright (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.Vector;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.LockSupport;
+
+import site.ycsb.wrappers.Comparison;
+import site.ycsb.wrappers.DatabaseField;
+
+/**
+ * Basic DB that just prints out the requested operations, instead of doing them against a database.
+ */
+public class BasicDB extends DB implements IndexableDB {
+ public static final String COUNT = "basicdb.count";
+ public static final String COUNT_DEFAULT = "false";
+
+ public static final String VERBOSE = "basicdb.verbose";
+ public static final String VERBOSE_DEFAULT = "true";
+
+ public static final String SIMULATE_DELAY = "basicdb.simulatedelay";
+ public static final String SIMULATE_DELAY_DEFAULT = "0";
+
+ public static final String RANDOMIZE_DELAY = "basicdb.randomizedelay";
+ public static final String RANDOMIZE_DELAY_DEFAULT = "true";
+
+ protected static final Object MUTEX = new Object();
+ protected static int counter = 0;
+ protected static Map reads;
+ protected static Map scans;
+ protected static Map updates;
+ protected static Map inserts;
+ protected static Map deletes;
+ protected static Map finds;
+
+ protected boolean verbose;
+ protected boolean randomizedelay;
+ protected int todelay;
+ protected boolean count;
+
+ public BasicDB() {
+ todelay = 0;
+ }
+
+ protected void delay() {
+ if (todelay > 0) {
+ long delayNs;
+ if (randomizedelay) {
+ delayNs = TimeUnit.MILLISECONDS.toNanos(ThreadLocalRandom.current().nextInt(todelay));
+ if (delayNs == 0) {
+ return;
+ }
+ } else {
+ delayNs = TimeUnit.MILLISECONDS.toNanos(todelay);
+ }
+
+ final long deadline = System.nanoTime() + delayNs;
+ do {
+ LockSupport.parkNanos(deadline - System.nanoTime());
+ } while (System.nanoTime() < deadline && !Thread.interrupted());
+ }
+ }
+
+ /**
+ * Initialize any state for this DB.
+ * Called once per DB instance; there is one DB instance per client thread.
+ */
+ public void init() {
+ verbose = Boolean.parseBoolean(getProperties().getProperty(VERBOSE, VERBOSE_DEFAULT));
+ todelay = Integer.parseInt(getProperties().getProperty(SIMULATE_DELAY, SIMULATE_DELAY_DEFAULT));
+ randomizedelay = Boolean.parseBoolean(getProperties().getProperty(RANDOMIZE_DELAY, RANDOMIZE_DELAY_DEFAULT));
+ count = Boolean.parseBoolean(getProperties().getProperty(COUNT, COUNT_DEFAULT));
+ if (verbose) {
+ synchronized (System.out) {
+ System.out.println("***************** properties *****************");
+ Properties p = getProperties();
+ if (p != null) {
+ for (Enumeration e = p.propertyNames(); e.hasMoreElements();) {
+ String k = (String) e.nextElement();
+ System.out.println("\"" + k + "\"=\"" + p.getProperty(k) + "\"");
+ }
+ }
+ System.out.println("**********************************************");
+ }
+ }
+
+ synchronized (MUTEX) {
+ if (counter == 0 && count) {
+ reads = new HashMap();
+ scans = new HashMap();
+ updates = new HashMap();
+ inserts = new HashMap();
+ deletes = new HashMap();
+ finds = new HashMap();
+ }
+ counter++;
+ }
+ }
+
+ protected static final ThreadLocal TL_STRING_BUILDER = new ThreadLocal() {
+ @Override
+ protected StringBuilder initialValue() {
+ return new StringBuilder();
+ }
+ };
+
+ protected static StringBuilder getStringBuilder() {
+ StringBuilder sb = TL_STRING_BUILDER.get();
+ sb.setLength(0);
+ return sb;
+ }
+
+ /**
+ * Read a record from the database. Each field/value pair from the result will be stored in a HashMap.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to read.
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A HashMap of field/value pairs for the result
+ * @return Zero on success, a non-zero error code on error
+ */
+ public Status read(String table, String key, Set fields, Map result) {
+ delay();
+
+ if (verbose) {
+ StringBuilder sb = getStringBuilder();
+ sb.append("READ ").append(table).append(" ").append(key).append(" [ ");
+ if (fields != null) {
+ for (String f : fields) {
+ sb.append(f).append(" ");
+ }
+ } else {
+ sb.append("");
+ }
+
+ sb.append("]");
+ System.out.println(sb);
+ }
+
+ if (count) {
+ incCounter(reads, hash(table, key, fields));
+ }
+
+ return Status.OK;
+ }
+
+ /**
+ * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored
+ * in a HashMap.
+ *
+ * @param table The name of the table
+ * @param startkey The record key of the first record to read.
+ * @param recordcount The number of records to read
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
+ * @return Zero on success, a non-zero error code on error
+ */
+ public Status scan(String table, String startkey, int recordcount, Set fields,
+ Vector> result) {
+ delay();
+
+ if (verbose) {
+ StringBuilder sb = getStringBuilder();
+ sb.append("SCAN ").append(table).append(" ").append(startkey).append(" ").append(recordcount).append(" [ ");
+ if (fields != null) {
+ for (String f : fields) {
+ sb.append(f).append(" ");
+ }
+ } else {
+ sb.append("");
+ }
+
+ sb.append("]");
+ System.out.println(sb);
+ }
+
+ if (count) {
+ incCounter(scans, hash(table, startkey, fields));
+ }
+
+ return Status.OK;
+ }
+
+ /**
+ * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the
+ * record with the specified record key, overwriting any existing values with the same field name.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to write.
+ * @param values A HashMap of field/value pairs to update in the record
+ * @return Zero on success, a non-zero error code on error
+ */
+ public Status update(String table, String key, Map values) {
+ delay();
+
+ if (verbose) {
+ StringBuilder sb = getStringBuilder();
+ sb.append("UPDATE ").append(table).append(" ").append(key).append(" [ ");
+ if (values != null) {
+ for (Map.Entry entry : values.entrySet()) {
+ sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" ");
+ }
+ }
+ sb.append("]");
+ System.out.println(sb);
+ }
+
+ if (count) {
+ incCounter(updates, oldHash(table, key, values));
+ }
+
+ return Status.OK;
+ }
+
+ /**
+ * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the
+ * record with the specified record key.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to insert.
+ * @param values A HashMap of field/value pairs to insert in the record
+ * @return Zero on success, a non-zero error code on error
+ */
+ public Status insert(String table, String key, List values) {
+ delay();
+
+ if (verbose) {
+ StringBuilder sb = getStringBuilder();
+ sb.append("INSERT ").append(table).append(" ").append(key).append(" [ ");
+ if (values != null) {
+ for (DatabaseField field : values) {
+ // for (Map.Entry entry : values.entrySet()) {
+ sb.append(field.getFieldname()).append("=").append(field.getContent().asIterator()).append(" ");
+ }
+ }
+
+ sb.append("]");
+ System.out.println(sb);
+ }
+
+ if (count) {
+ incCounter(inserts, hashWithDatabaseField(table, key, values));
+ }
+
+ return Status.OK;
+ }
+
+
+ /**
+ * Delete a record from the database.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to delete.
+ * @return Zero on success, a non-zero error code on error
+ */
+ public Status delete(String table, String key) {
+ delay();
+
+ if (verbose) {
+ StringBuilder sb = getStringBuilder();
+ sb.append("DELETE ").append(table).append(" ").append(key);
+ System.out.println(sb);
+ }
+
+ if (count) {
+ incCounter(deletes, (table + key).hashCode());
+ }
+
+ return Status.OK;
+ }
+
+ @Override
+ public void cleanup() {
+ synchronized (MUTEX) {
+ int countDown = --counter;
+ if (count && countDown < 1) {
+ // TODO - would be nice to call something like:
+ // Measurements.getMeasurements().oneOffMeasurement("READS", "Uniques", reads.size());
+ System.out.println("[READS], Uniques, " + reads.size());
+ System.out.println("[SCANS], Uniques, " + scans.size());
+ System.out.println("[UPDATES], Uniques, " + updates.size());
+ System.out.println("[INSERTS], Uniques, " + inserts.size());
+ System.out.println("[DELETES], Uniques, " + deletes.size());
+ }
+ }
+ }
+
+ /**
+ * Increments the count on the hash in the map.
+ * @param map A non-null map to sync and use for incrementing.
+ * @param hash A hash code to increment.
+ */
+ protected void incCounter(final Map map, final int hash) {
+ synchronized (map) {
+ Integer ctr = map.get(hash);
+ if (ctr == null) {
+ map.put(hash, 1);
+ } else {
+ map.put(hash, ctr + 1);
+ }
+ }
+ }
+
+ /**
+ * Hashes the table, key and fields, sorting the fields first for a consistent
+ * hash.
+ * Note that this is expensive as we generate a copy of the fields and a string
+ * buffer to hash on. Hashing on the objects is problematic.
+ * @param table The user table.
+ * @param key The key read or scanned.
+ * @param fields The fields read or scanned.
+ * @return The hash code.
+ */
+ protected int hash(final String table, final String key, final Set fields) {
+ if (fields == null) {
+ return (table + key).hashCode();
+ }
+ StringBuilder buf = getStringBuilder().append(table).append(key);
+ List sorted = new ArrayList(fields);
+ Collections.sort(sorted);
+ for (final String field : sorted) {
+ buf.append(field);
+ }
+ return buf.toString().hashCode();
+ }
+
+ /**
+ * Hashes the table, key and fields, sorting the fields first for a consistent
+ * hash.
+ * Note that this is expensive as we generate a copy of the fields and a string
+ * buffer to hash on. Hashing on the objects is problematic.
+ * @param table The user table.
+ * @param key The key read or scanned.
+ * @param values The values to hash on.
+ * @return The hash code.
+ */
+ protected int oldHash(final String table, final String key, final Map values) {
+ if (values == null) {
+ return (table + key).hashCode();
+ }
+ final TreeMap sorted =
+ new TreeMap(values);
+
+ StringBuilder buf = getStringBuilder().append(table).append(key);
+ for (final Entry entry : sorted.entrySet()) {
+ entry.getValue().reset();
+ buf.append(entry.getKey())
+ .append(entry.getValue().toString());
+ }
+ return buf.toString().hashCode();
+ }
+
+ protected int hash(final String table, final String key, final List values) {
+ if (values == null) {
+ return (table + key).hashCode();
+ }
+ List sorted = new ArrayList<>(values);
+ sorted.sort((el1, el2) -> { return el1.getFieldname().compareTo(el2.getFieldname()); });
+
+ StringBuilder buf = getStringBuilder().append(table).append(key);
+ for (final Comparison field : sorted) {
+ buf.append(field.getFieldname())
+ .append(field.getContentAsString());
+ }
+ return buf.toString().hashCode();
+ }
+
+ protected int hashWithDatabaseField(final String table, final String key, final List values) {
+ if (values == null) {
+ return (table + key).hashCode();
+ }
+ List sorted = new ArrayList<>(values);
+ sorted.sort((el1, el2) -> { return el1.getFieldname().compareTo(el2.getFieldname()); });
+
+ StringBuilder buf = getStringBuilder().append(table).append(key);
+ for (final DatabaseField field : sorted) {
+ ByteIterator b = field.getContent().asIterator();
+ b.reset();
+ buf.append(field.getFieldname())
+ .append(b.toString());
+ }
+ return buf.toString().hashCode();
+ }
+
+ /**
+ * Find a record from the database.
+ *
+ * @param table The name of the table
+ * @param key The values for the filter
+ */
+ @Override
+ public Status findOne(String table, List filters,
+ Set fields, Map result) {
+ delay();
+
+ if (verbose) {
+ StringBuilder sb = getStringBuilder();
+ sb.append("FINDONE ").append(table).append(" ").append(filters)
+ .append(" -> ").append(fields);
+ System.out.println(sb);
+ }
+
+ if (count) {
+ incCounter(finds, hash(table, "", filters));
+ }
+
+ return Status.OK;
+ }
+
+ @Override
+ public Status updateOne(String table, List filters, List fields) {
+ delay();
+ if (verbose) {
+ StringBuilder sb = getStringBuilder();
+ sb.append("UPDATEONE ").append(table).append(" ").append(filters)
+ .append(" -> ").append(fields);
+ System.out.println(sb);
+ }
+
+ if (count) {
+ incCounter(finds, hash(table, "", filters));
+ }
+
+ return Status.OK;
+ }
+ /**
+ * Short test of BasicDB
+ */
+ /*
+ public static void main(String[] args) {
+ BasicDB bdb = new BasicDB();
+
+ Properties p = new Properties();
+ p.setProperty("Sky", "Blue");
+ p.setProperty("Ocean", "Wet");
+
+ bdb.setProperties(p);
+
+ bdb.init();
+
+ HashMap fields = new HashMap();
+ fields.put("A", new StringByteIterator("X"));
+ fields.put("B", new StringByteIterator("Y"));
+
+ bdb.read("table", "key", null, null);
+ bdb.insert("table", "key", fields);
+
+ fields = new HashMap();
+ fields.put("C", new StringByteIterator("Z"));
+
+ bdb.update("table", "key", fields);
+
+ bdb.delete("table", "key");
+ }
+ */
+}
diff --git a/core/src/main/java/site/ycsb/BasicTSDB.java b/core/src/main/java/site/ycsb/BasicTSDB.java
new file mode 100644
index 0000000..b3f1016
--- /dev/null
+++ b/core/src/main/java/site/ycsb/BasicTSDB.java
@@ -0,0 +1,277 @@
+/**
+ * Copyright (c) 2017 YCSB contributors All rights reserved.
+ * Copyright (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+import site.ycsb.workloads.TimeSeriesWorkload;
+import site.ycsb.wrappers.DatabaseField;
+
+/**
+ * Basic DB for printing out time series workloads and/or tracking the distribution
+ * of keys and fields.
+ */
+public class BasicTSDB extends BasicDB {
+
+ /** Time series workload specific counters. */
+ protected static Map timestamps;
+ protected static Map floats;
+ protected static Map integers;
+
+ private String timestampKey;
+ private String valueKey;
+ private String tagPairDelimiter;
+ private String queryTimeSpanDelimiter;
+ private long lastTimestamp;
+
+ @Override
+ public void init() {
+ super.init();
+
+ synchronized (MUTEX) {
+ if (timestamps == null) {
+ timestamps = new HashMap();
+ floats = new HashMap();
+ integers = new HashMap();
+ }
+ }
+
+ timestampKey = getProperties().getProperty(
+ TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY,
+ TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT);
+ valueKey = getProperties().getProperty(
+ TimeSeriesWorkload.VALUE_KEY_PROPERTY,
+ TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT);
+ tagPairDelimiter = getProperties().getProperty(
+ TimeSeriesWorkload.PAIR_DELIMITER_PROPERTY,
+ TimeSeriesWorkload.PAIR_DELIMITER_PROPERTY_DEFAULT);
+ queryTimeSpanDelimiter = getProperties().getProperty(
+ TimeSeriesWorkload.QUERY_TIMESPAN_DELIMITER_PROPERTY,
+ TimeSeriesWorkload.QUERY_TIMESPAN_DELIMITER_PROPERTY_DEFAULT);
+ }
+
+ public Status read(String table, String key, Set fields, Map result) {
+ delay();
+
+ if (verbose) {
+ StringBuilder sb = getStringBuilder();
+ sb.append("READ ").append(table).append(" ").append(key).append(" [ ");
+ if (fields != null) {
+ for (String f : fields) {
+ sb.append(f).append(" ");
+ }
+ } else {
+ sb.append("");
+ }
+
+ sb.append("]");
+ System.out.println(sb);
+ }
+
+ if (count) {
+ Set filtered = null;
+ if (fields != null) {
+ filtered = new HashSet();
+ for (final String field : fields) {
+ if (field.startsWith(timestampKey)) {
+ String[] parts = field.split(tagPairDelimiter);
+ if (parts[1].contains(queryTimeSpanDelimiter)) {
+ parts = parts[1].split(queryTimeSpanDelimiter);
+ lastTimestamp = Long.parseLong(parts[0]);
+ } else {
+ lastTimestamp = Long.parseLong(parts[1]);
+ }
+ synchronized(timestamps) {
+ Integer ctr = timestamps.get(lastTimestamp);
+ if (ctr == null) {
+ timestamps.put(lastTimestamp, 1);
+ } else {
+ timestamps.put(lastTimestamp, ctr + 1);
+ }
+ }
+ } else {
+ filtered.add(field);
+ }
+ }
+ }
+ incCounter(reads, hash(table, key, filtered));
+ }
+ return Status.OK;
+ }
+
+ @Override
+ public Status update(String table, String key, Map values) {
+ delay();
+
+ boolean isFloat = false;
+
+ if (verbose) {
+ StringBuilder sb = getStringBuilder();
+ sb.append("UPDATE ").append(table).append(" ").append(key).append(" [ ");
+ if (values != null) {
+ final TreeMap tree = new TreeMap(values);
+ for (Map.Entry entry : tree.entrySet()) {
+ if (entry.getKey().equals(timestampKey)) {
+ sb.append(entry.getKey()).append("=")
+ .append(Utils.bytesToLong(entry.getValue().toArray())).append(" ");
+ } else if (entry.getKey().equals(valueKey)) {
+ final NumericByteIterator it = (NumericByteIterator) entry.getValue();
+ isFloat = it.isFloatingPoint();
+ sb.append(entry.getKey()).append("=")
+ .append(isFloat ? it.getDouble() : it.getLong()).append(" ");
+ } else {
+ sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" ");
+ }
+ }
+ }
+ sb.append("]");
+ System.out.println(sb);
+ }
+
+ if (count) {
+ if (!verbose) {
+ isFloat = ((NumericByteIterator) values.get(valueKey)).isFloatingPoint();
+ }
+ int hash = hash(table, key, values);
+ incCounter(updates, hash);
+ synchronized(timestamps) {
+ Integer ctr = timestamps.get(lastTimestamp);
+ if (ctr == null) {
+ timestamps.put(lastTimestamp, 1);
+ } else {
+ timestamps.put(lastTimestamp, ctr + 1);
+ }
+ }
+ if (isFloat) {
+ incCounter(floats, hash);
+ } else {
+ incCounter(integers, hash);
+ }
+ }
+
+ return Status.OK;
+ }
+
+ @Override
+ public Status insert(String table, String key, List values_) {
+ Map values = DB.fieldListAsIteratorMap(values_);
+ delay();
+
+ boolean isFloat = false;
+
+ if (verbose) {
+ StringBuilder sb = getStringBuilder();
+ sb.append("INSERT ").append(table).append(" ").append(key).append(" [ ");
+ if (values != null) {
+ final TreeMap tree = new TreeMap(values);
+ for (Map.Entry entry : tree.entrySet()) {
+ if (entry.getKey().equals(timestampKey)) {
+ sb.append(entry.getKey()).append("=")
+ .append(Utils.bytesToLong(entry.getValue().toArray())).append(" ");
+ } else if (entry.getKey().equals(valueKey)) {
+ final NumericByteIterator it = (NumericByteIterator) entry.getValue();
+ isFloat = it.isFloatingPoint();
+ sb.append(entry.getKey()).append("=")
+ .append(isFloat ? it.getDouble() : it.getLong()).append(" ");
+ } else {
+ sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" ");
+ }
+ }
+ }
+ sb.append("]");
+ System.out.println(sb);
+ }
+
+ if (count) {
+ if (!verbose) {
+ isFloat = ((NumericByteIterator) values.get(valueKey)).isFloatingPoint();
+ }
+ int hash = hash(table, key, values);
+ incCounter(inserts, hash);
+ synchronized(timestamps) {
+ Integer ctr = timestamps.get(lastTimestamp);
+ if (ctr == null) {
+ timestamps.put(lastTimestamp, 1);
+ } else {
+ timestamps.put(lastTimestamp, ctr + 1);
+ }
+ }
+ if (isFloat) {
+ incCounter(floats, hash);
+ } else {
+ incCounter(integers, hash);
+ }
+ }
+
+ return Status.OK;
+ }
+
+ @Override
+ public void cleanup() {
+ super.cleanup();
+ if (count && counter < 1) {
+ System.out.println("[TIMESTAMPS], Unique, " + timestamps.size());
+ System.out.println("[FLOATS], Unique series, " + floats.size());
+ System.out.println("[INTEGERS], Unique series, " + integers.size());
+
+ long minTs = Long.MAX_VALUE;
+ long maxTs = Long.MIN_VALUE;
+ for (final long ts : timestamps.keySet()) {
+ if (ts > maxTs) {
+ maxTs = ts;
+ }
+ if (ts < minTs) {
+ minTs = ts;
+ }
+ }
+ System.out.println("[TIMESTAMPS], Min, " + minTs);
+ System.out.println("[TIMESTAMPS], Max, " + maxTs);
+ }
+ }
+
+
+ protected int hash(final String table, final String key, final Map values) {
+ final TreeMap sorted = new TreeMap();
+ for (final Entry entry : values.entrySet()) {
+ if (entry.getKey().equals(valueKey)) {
+ continue;
+ } else if (entry.getKey().equals(timestampKey)) {
+ lastTimestamp = ((NumericByteIterator) entry.getValue()).getLong();
+ entry.getValue().reset();
+ continue;
+ }
+ sorted.put(entry.getKey(), entry.getValue());
+ }
+ // yeah it's ugly but gives us a unique hash without having to add hashers
+ // to all of the ByteIterators.
+ StringBuilder buf = new StringBuilder().append(table).append(key);
+ for (final Entry entry : sorted.entrySet()) {
+ entry.getValue().reset();
+ buf.append(entry.getKey())
+ .append(entry.getValue().toString());
+ }
+ return buf.toString().hashCode();
+ }
+
+}
\ No newline at end of file
diff --git a/core/src/main/java/site/ycsb/ByteArrayByteIterator.java b/core/src/main/java/site/ycsb/ByteArrayByteIterator.java
new file mode 100644
index 0000000..b27c808
--- /dev/null
+++ b/core/src/main/java/site/ycsb/ByteArrayByteIterator.java
@@ -0,0 +1,73 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb;
+
+/**
+ * A ByteIterator that iterates through a byte array.
+ */
+public class ByteArrayByteIterator extends ByteIterator {
+ private final int originalOffset;
+ private final byte[] str;
+ private int off;
+ private final int len;
+
+ public ByteArrayByteIterator(byte[] s) {
+ this.str = s;
+ this.off = 0;
+ this.len = s.length;
+ originalOffset = 0;
+ }
+
+ public ByteArrayByteIterator(byte[] s, int off, int len) {
+ this.str = s;
+ this.off = off;
+ this.len = off + len;
+ originalOffset = off;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return off < len;
+ }
+
+ @Override
+ public byte nextByte() {
+ byte ret = str[off];
+ off++;
+ return ret;
+ }
+
+ @Override
+ public long bytesLeft() {
+ return len - off;
+ }
+
+ @Override
+ public void reset() {
+ off = originalOffset;
+ }
+
+ @Override
+ public byte[] toArray() {
+ int size = (int) bytesLeft();
+ byte[] bytes = new byte[size];
+ System.arraycopy(str, off, bytes, 0, size);
+ off = len;
+ return bytes;
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/ByteIterator.java b/core/src/main/java/site/ycsb/ByteIterator.java
new file mode 100644
index 0000000..768f381
--- /dev/null
+++ b/core/src/main/java/site/ycsb/ByteIterator.java
@@ -0,0 +1,105 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.util.Iterator;
+
+/**
+ * YCSB-specific buffer class. ByteIterators are designed to support
+ * efficient field generation, and to allow backend drivers that can stream
+ * fields (instead of materializing them in RAM) to do so.
+ *
+ * YCSB originially used String objects to represent field values. This led to
+ * two performance issues.
+ *
+ * First, it leads to unnecessary conversions between UTF-16 and UTF-8, both
+ * during field generation, and when passing data to byte-based backend
+ * drivers.
+ *
+ * Second, Java strings are represented internally using UTF-16, and are
+ * built by appending to a growable array type (StringBuilder or
+ * StringBuffer), then calling a toString() method. This leads to a 4x memory
+ * overhead as field values are being built, which prevented YCSB from
+ * driving large object stores.
+ *
+ * The StringByteIterator class contains a number of convenience methods for
+ * backend drivers that convert between Map<String,String> and
+ * Map<String,ByteBuffer>.
+ *
+ */
+public abstract class ByteIterator implements Iterator {
+
+ @Override
+ public abstract boolean hasNext();
+
+ @Override
+ public Byte next() {
+ throw new UnsupportedOperationException();
+ }
+
+ public abstract byte nextByte();
+
+ /** @return byte offset immediately after the last valid byte */
+ public int nextBuf(byte[] buf, int bufOff) {
+ int sz = bufOff;
+ while (sz < buf.length && hasNext()) {
+ buf[sz] = nextByte();
+ sz++;
+ }
+ return sz;
+ }
+
+ public abstract long bytesLeft();
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** Resets the iterator so that it can be consumed again. Not all
+ * implementations support this call.
+ * @throws UnsupportedOperationException if the implementation hasn't implemented
+ * the method.
+ */
+ public void reset() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** Consumes remaining contents of this object, and returns them as a string. */
+ public String toString() {
+ Charset cset = Charset.forName("UTF-8");
+ CharBuffer cb = cset.decode(ByteBuffer.wrap(this.toArray()));
+ return cb.toString();
+ }
+
+ /** Consumes remaining contents of this object, and returns them as a byte array. */
+ public byte[] toArray() {
+ long left = bytesLeft();
+ if (left != (int) left) {
+ throw new ArrayIndexOutOfBoundsException("Too much data to fit in one array!");
+ }
+ byte[] ret = new byte[(int) left];
+ for (int i = 0; i < ret.length; i++) {
+ ret[i] = nextByte();
+ }
+ return ret;
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/Client.java b/core/src/main/java/site/ycsb/Client.java
new file mode 100644
index 0000000..7240616
--- /dev/null
+++ b/core/src/main/java/site/ycsb/Client.java
@@ -0,0 +1,680 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ * Copyright (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import site.ycsb.measurements.Measurements;
+import site.ycsb.measurements.exporter.MeasurementsExporter;
+import site.ycsb.measurements.exporter.TextMeasurementsExporter;
+import org.apache.htrace.core.HTraceConfiguration;
+import org.apache.htrace.core.TraceScope;
+import org.apache.htrace.core.Tracer;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Turn seconds remaining into more useful units.
+ * i.e. if there are hours or days worth of seconds, use them.
+ */
+final class RemainingFormatter {
+ private RemainingFormatter() {
+ // not used
+ }
+
+ public static StringBuilder format(long seconds) {
+ StringBuilder time = new StringBuilder();
+ long days = TimeUnit.SECONDS.toDays(seconds);
+ if (days > 0) {
+ time.append(days).append(days == 1 ? " day " : " days ");
+ seconds -= TimeUnit.DAYS.toSeconds(days);
+ }
+ long hours = TimeUnit.SECONDS.toHours(seconds);
+ if (hours > 0) {
+ time.append(hours).append(hours == 1 ? " hour " : " hours ");
+ seconds -= TimeUnit.HOURS.toSeconds(hours);
+ }
+ /* Only include minute granularity if we're < 1 day. */
+ if (days < 1) {
+ long minutes = TimeUnit.SECONDS.toMinutes(seconds);
+ if (minutes > 0) {
+ time.append(minutes).append(minutes == 1 ? " minute " : " minutes ");
+ seconds -= TimeUnit.MINUTES.toSeconds(seconds);
+ }
+ }
+ /* Only bother to include seconds if we're < 1 minute */
+ if (time.length() == 0) {
+ time.append(seconds).append(time.length() == 1 ? " second " : " seconds ");
+ }
+ return time;
+ }
+}
+
+/**
+ * Main class for executing YCSB.
+ */
+public final class Client {
+ private Client() {
+ //not used
+ }
+
+ public static final String DEFAULT_RECORD_COUNT = "0";
+
+ /**
+ * The target number of operations to perform.
+ */
+ public static final String OPERATION_COUNT_PROPERTY = "operationcount";
+
+ /**
+ * The number of records to load into the database initially.
+ */
+ public static final String RECORD_COUNT_PROPERTY = "recordcount";
+
+ /**
+ * The workload class to be loaded.
+ */
+ public static final String WORKLOAD_PROPERTY = "workload";
+
+ /**
+ * The database class to be used.
+ */
+ public static final String DB_PROPERTY = "db";
+
+ /**
+ * The exporter class to be used. The default is
+ * site.ycsb.measurements.exporter.TextMeasurementsExporter.
+ */
+ public static final String EXPORTER_PROPERTY = "exporter";
+
+ /**
+ * If set to the path of a file, YCSB will write all output to this file
+ * instead of STDOUT.
+ */
+ public static final String EXPORT_FILE_PROPERTY = "exportfile";
+
+ /**
+ * The number of YCSB client threads to run.
+ */
+ public static final String THREAD_COUNT_PROPERTY = "threadcount";
+
+ /**
+ * Indicates how many inserts to do if less than recordcount.
+ * Useful for partitioning the load among multiple servers if the client is the bottleneck.
+ * Additionally workloads should support the "insertstart" property which tells them which record to start at.
+ */
+ public static final String INSERT_COUNT_PROPERTY = "insertcount";
+
+ /**
+ * Target number of operations per second.
+ */
+ public static final String TARGET_PROPERTY = "target";
+
+ /**
+ * The maximum amount of time (in seconds) for which the benchmark will be run.
+ */
+ public static final String MAX_EXECUTION_TIME = "maxexecutiontime";
+
+ /**
+ * Whether or not this is the transaction phase (run) or not (load).
+ */
+ public static final String DO_TRANSACTIONS_PROPERTY = "dotransactions";
+
+ /**
+ * Whether or not to show status during run.
+ */
+ public static final String STATUS_PROPERTY = "status";
+
+ /**
+ * Use label for status (e.g. to label one experiment out of a whole batch).
+ */
+ public static final String LABEL_PROPERTY = "label";
+
+ /**
+ * An optional thread used to track progress and measure JVM stats.
+ */
+ private static StatusThread statusthread = null;
+
+ // HTrace integration related constants.
+
+ /**
+ * All keys for configuring the tracing system start with this prefix.
+ */
+ private static final String HTRACE_KEY_PREFIX = "htrace.";
+ private static final String CLIENT_WORKLOAD_INIT_SPAN = "Client#workload_init";
+ private static final String CLIENT_INIT_SPAN = "Client#init";
+ private static final String CLIENT_WORKLOAD_SPAN = "Client#workload";
+ private static final String CLIENT_CLEANUP_SPAN = "Client#cleanup";
+ private static final String CLIENT_EXPORT_MEASUREMENTS_SPAN = "Client#export_measurements";
+
+ public static void usageMessage() {
+ System.out.println("Usage: java site.ycsb.Client [options]");
+ System.out.println("Options:");
+ System.out.println(" -threads n: execute using n threads (default: 1) - can also be specified as the \n" +
+ " \"threadcount\" property using -p");
+ System.out.println(" -target n: attempt to do n operations per second (default: unlimited) - can also\n" +
+ " be specified as the \"target\" property using -p");
+ System.out.println(" -load: run the loading phase of the workload");
+ System.out.println(" -t: run the transactions phase of the workload (default)");
+ System.out.println(" -db dbname: specify the name of the DB to use (default: site.ycsb.BasicDB) - \n" +
+ " can also be specified as the \"db\" property using -p");
+ System.out.println(" -P propertyfile: load properties from the given file. Multiple files can");
+ System.out.println(" be specified, and will be processed in the order specified");
+ System.out.println(" -p name=value: specify a property to be passed to the DB and workloads;");
+ System.out.println(" multiple properties can be specified, and override any");
+ System.out.println(" values in the propertyfile");
+ System.out.println(" -s: show status during run (default: no status)");
+ System.out.println(" -l label: use label for status (e.g. to label one experiment out of a whole batch)");
+ System.out.println("");
+ System.out.println("Required properties:");
+ System.out.println(" " + WORKLOAD_PROPERTY + ": the name of the workload class to use (e.g. " +
+ "site.ycsb.workloads.CoreWorkload)");
+ System.out.println("");
+ System.out.println("To run the transaction phase from multiple servers, start a separate client on each.");
+ System.out.println("To run the load phase from multiple servers, start a separate client on each; additionally,");
+ System.out.println("use the \"insertcount\" and \"insertstart\" properties to divide up the records " +
+ "to be inserted");
+ }
+
+ public static boolean checkRequiredProperties(Properties props) {
+ if (props.getProperty(WORKLOAD_PROPERTY) == null) {
+ System.out.println("Missing property: " + WORKLOAD_PROPERTY);
+ return false;
+ }
+
+ return true;
+ }
+
+
+ /**
+ * Exports the measurements to either sysout or a file using the exporter
+ * loaded from conf.
+ *
+ * @throws IOException Either failed to write to output stream or failed to close it.
+ */
+ private static void exportMeasurements(Properties props, long opcount, long runtime)
+ throws IOException {
+ MeasurementsExporter exporter = null;
+ try {
+ // if no destination file is provided the results will be written to stdout
+ OutputStream out;
+ String exportFile = props.getProperty(EXPORT_FILE_PROPERTY);
+ if (exportFile == null) {
+ out = System.out;
+ } else {
+ out = new FileOutputStream(exportFile);
+ }
+
+ // if no exporter is provided the default text one will be used
+ String exporterStr = props.getProperty(EXPORTER_PROPERTY,
+ "site.ycsb.measurements.exporter.TextMeasurementsExporter");
+ try {
+ exporter = (MeasurementsExporter) Class.forName(exporterStr).getConstructor(OutputStream.class)
+ .newInstance(out);
+ } catch (Exception e) {
+ System.err.println("Could not find exporter " + exporterStr
+ + ", will use default text reporter.");
+ e.printStackTrace();
+ exporter = new TextMeasurementsExporter(out);
+ }
+
+ exporter.write("OVERALL", "RunTime(ms)", runtime);
+ double throughput = 1000.0 * (opcount) / (runtime);
+ exporter.write("OVERALL", "Throughput(ops/sec)", throughput);
+
+ final Map gcs = Utils.getGCStatst();
+ long totalGCCount = 0;
+ long totalGCTime = 0;
+ for (final Entry entry : gcs.entrySet()) {
+ exporter.write("TOTAL_GCS_" + entry.getKey(), "Count", entry.getValue()[0]);
+ exporter.write("TOTAL_GC_TIME_" + entry.getKey(), "Time(ms)", entry.getValue()[1]);
+ exporter.write("TOTAL_GC_TIME_%_" + entry.getKey(), "Time(%)",
+ ((double) entry.getValue()[1] / runtime) * (double) 100);
+ totalGCCount += entry.getValue()[0];
+ totalGCTime += entry.getValue()[1];
+ }
+ exporter.write("TOTAL_GCs", "Count", totalGCCount);
+
+ exporter.write("TOTAL_GC_TIME", "Time(ms)", totalGCTime);
+ exporter.write("TOTAL_GC_TIME_%", "Time(%)", ((double) totalGCTime / runtime) * (double) 100);
+ if (statusthread != null && statusthread.trackJVMStats()) {
+ exporter.write("MAX_MEM_USED", "MBs", statusthread.getMaxUsedMem());
+ exporter.write("MIN_MEM_USED", "MBs", statusthread.getMinUsedMem());
+ exporter.write("MAX_THREADS", "Count", statusthread.getMaxThreads());
+ exporter.write("MIN_THREADS", "Count", statusthread.getMinThreads());
+ exporter.write("MAX_SYS_LOAD_AVG", "Load", statusthread.getMaxLoadAvg());
+ exporter.write("MIN_SYS_LOAD_AVG", "Load", statusthread.getMinLoadAvg());
+ }
+
+ Measurements.getMeasurements().exportMeasurements(exporter);
+ } finally {
+ if (exporter != null) {
+ exporter.close();
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static void main(String[] args) {
+ Properties props = parseArguments(args);
+
+ boolean status = Boolean.valueOf(props.getProperty(STATUS_PROPERTY, String.valueOf(false)));
+ String label = props.getProperty(LABEL_PROPERTY, "");
+
+ long maxExecutionTime = Integer.parseInt(props.getProperty(MAX_EXECUTION_TIME, "0"));
+
+ //get number of threads, target and db
+ int threadcount = Integer.parseInt(props.getProperty(THREAD_COUNT_PROPERTY, "1"));
+ String dbname = props.getProperty(DB_PROPERTY, "site.ycsb.BasicDB");
+ int target = Integer.parseInt(props.getProperty(TARGET_PROPERTY, "0"));
+
+ //compute the target throughput
+ double targetperthreadperms = -1;
+ if (target > 0) {
+ double targetperthread = ((double) target) / ((double) threadcount);
+ targetperthreadperms = targetperthread / 1000.0;
+ }
+
+ Thread warningthread = setupWarningThread();
+ warningthread.start();
+
+ Measurements.setProperties(props);
+
+ Workload workload = getWorkload(props);
+
+ final Tracer tracer = getTracer(props, workload);
+
+ initWorkload(props, warningthread, workload, tracer);
+
+ System.err.println("Starting test.");
+ final CountDownLatch completeLatch = new CountDownLatch(threadcount);
+
+ final List clients = initDb(dbname, props, threadcount, targetperthreadperms,
+ workload, tracer, completeLatch);
+
+ if (status) {
+ boolean standardstatus = false;
+ if (props.getProperty(Measurements.MEASUREMENT_TYPE_PROPERTY, "").compareTo("timeseries") == 0) {
+ standardstatus = true;
+ }
+ int statusIntervalSeconds = Integer.parseInt(props.getProperty("status.interval", "10"));
+ boolean trackJVMStats = props.getProperty(Measurements.MEASUREMENT_TRACK_JVM_PROPERTY,
+ Measurements.MEASUREMENT_TRACK_JVM_PROPERTY_DEFAULT).equals("true");
+ statusthread = new StatusThread(completeLatch, clients, label, standardstatus, statusIntervalSeconds,
+ trackJVMStats);
+ statusthread.start();
+ }
+
+ Thread terminator = null;
+ long st;
+ long en;
+ long opsDone;
+
+ try (final TraceScope span = tracer.newScope(CLIENT_WORKLOAD_SPAN)) {
+
+ final Map threads = new HashMap<>(threadcount);
+ for (ClientThread client : clients) {
+ threads.put(new Thread(tracer.wrap(client, "ClientThread")), client);
+ }
+
+ st = System.currentTimeMillis();
+
+ for (Thread t : threads.keySet()) {
+ t.start();
+ }
+
+ if (maxExecutionTime > 0) {
+ terminator = new TerminatorThread(maxExecutionTime, threads.keySet(), workload);
+ terminator.start();
+ }
+
+ opsDone = 0;
+
+ for (Map.Entry entry : threads.entrySet()) {
+ try {
+ entry.getKey().join();
+ opsDone += entry.getValue().getOpsDone();
+ } catch (InterruptedException ignored) {
+ // ignored
+ }
+ }
+
+ en = System.currentTimeMillis();
+ }
+
+ try {
+ try (final TraceScope span = tracer.newScope(CLIENT_CLEANUP_SPAN)) {
+
+ if (terminator != null && !terminator.isInterrupted()) {
+ terminator.interrupt();
+ }
+
+ if (status) {
+ // wake up status thread if it's asleep
+ statusthread.interrupt();
+ // at this point we assume all the monitored threads are already gone as per above join loop.
+ try {
+ statusthread.join();
+ } catch (InterruptedException ignored) {
+ // ignored
+ }
+ }
+
+ workload.cleanup();
+ }
+ } catch (WorkloadException e) {
+ e.printStackTrace();
+ e.printStackTrace(System.out);
+ System.exit(0);
+ }
+
+ try {
+ try (final TraceScope span = tracer.newScope(CLIENT_EXPORT_MEASUREMENTS_SPAN)) {
+ exportMeasurements(props, opsDone, en - st);
+ }
+ } catch (IOException e) {
+ System.err.println("Could not export measurements, error: " + e.getMessage());
+ e.printStackTrace();
+ System.exit(-1);
+ }
+
+ System.exit(0);
+ }
+
+ private static List initDb(String dbname, Properties props, int threadcount,
+ double targetperthreadperms, Workload workload, Tracer tracer,
+ CountDownLatch completeLatch) {
+ boolean initFailed = false;
+ boolean dotransactions = Boolean.valueOf(props.getProperty(DO_TRANSACTIONS_PROPERTY, String.valueOf(true)));
+
+ final List clients = new ArrayList<>(threadcount);
+ try (final TraceScope span = tracer.newScope(CLIENT_INIT_SPAN)) {
+ long opcount;
+ if (dotransactions) {
+ opcount = Long.parseLong(props.getProperty(OPERATION_COUNT_PROPERTY, "0"));
+ } else {
+ if (props.containsKey(INSERT_COUNT_PROPERTY)) {
+ opcount = Long.parseLong(props.getProperty(INSERT_COUNT_PROPERTY, "0"));
+ } else {
+ opcount = Long.parseLong(props.getProperty(RECORD_COUNT_PROPERTY, DEFAULT_RECORD_COUNT));
+ }
+ }
+ if (threadcount > opcount && opcount > 0){
+ threadcount = (int) opcount;
+ System.out.println("Warning: the threadcount is bigger than recordcount, the threadcount will be recordcount!");
+ }
+ for (int threadid = 0; threadid < threadcount; threadid++) {
+ DB db;
+ try {
+ db = DBFactory.newDB(dbname, props, tracer);
+ } catch (UnknownDBException e) {
+ System.out.println("Unknown DB " + dbname);
+ initFailed = true;
+ break;
+ }
+
+ long threadopcount = opcount / threadcount;
+
+ // ensure correct number of operations, in case opcount is not a multiple of threadcount
+ if (threadid < opcount % threadcount) {
+ ++threadopcount;
+ }
+
+ ClientThread t = new ClientThread(db, dotransactions, workload, props, threadopcount, targetperthreadperms,
+ completeLatch);
+ t.setThreadId(threadid);
+ t.setThreadCount(threadcount);
+ clients.add(t);
+ }
+
+ if (initFailed) {
+ System.err.println("Error initializing datastore bindings.");
+ System.exit(0);
+ }
+ }
+ return clients;
+ }
+
+ private static Tracer getTracer(Properties props, Workload workload) {
+ return new Tracer.Builder("YCSB " + workload.getClass().getSimpleName())
+ .conf(getHTraceConfiguration(props))
+ .build();
+ }
+
+ private static void initWorkload(Properties props, Thread warningthread, Workload workload, Tracer tracer) {
+ try {
+ try (final TraceScope span = tracer.newScope(CLIENT_WORKLOAD_INIT_SPAN)) {
+ workload.init(props);
+ warningthread.interrupt();
+ }
+ } catch (WorkloadException e) {
+ e.printStackTrace();
+ e.printStackTrace(System.out);
+ System.exit(0);
+ }
+ }
+
+ private static HTraceConfiguration getHTraceConfiguration(Properties props) {
+ final Map filteredProperties = new HashMap<>();
+ for (String key : props.stringPropertyNames()) {
+ if (key.startsWith(HTRACE_KEY_PREFIX)) {
+ filteredProperties.put(key.substring(HTRACE_KEY_PREFIX.length()), props.getProperty(key));
+ }
+ }
+ return HTraceConfiguration.fromMap(filteredProperties);
+ }
+
+ private static Thread setupWarningThread() {
+ //show a warning message that creating the workload is taking a while
+ //but only do so if it is taking longer than 2 seconds
+ //(showing the message right away if the setup wasn't taking very long was confusing people)
+ return new Thread() {
+ @Override
+ public void run() {
+ try {
+ sleep(2000);
+ } catch (InterruptedException e) {
+ return;
+ }
+ System.err.println(" (might take a few minutes for large data sets)");
+ }
+ };
+ }
+
+ private static Workload getWorkload(Properties props) {
+ ClassLoader classLoader = Client.class.getClassLoader();
+
+ try {
+ Properties projectProp = new Properties();
+ projectProp.load(classLoader.getResourceAsStream("project.properties"));
+ System.err.println("YCSB Client " + projectProp.getProperty("version"));
+ } catch (IOException e) {
+ System.err.println("Unable to retrieve client version.");
+ }
+
+ System.err.println();
+ System.err.println("Loading workload...");
+ try {
+ Class workloadclass = classLoader.loadClass(props.getProperty(WORKLOAD_PROPERTY));
+
+ return (Workload) workloadclass.newInstance();
+ } catch (Exception e) {
+ e.printStackTrace();
+ e.printStackTrace(System.out);
+ System.exit(0);
+ }
+
+ return null;
+ }
+
+ private static Properties parseArguments(String[] args) {
+ Properties props = new Properties();
+ System.err.print("Command line:");
+ for (String arg : args) {
+ System.err.print(" " + arg);
+ }
+ System.err.println();
+
+ Properties fileprops = new Properties();
+ int argindex = 0;
+
+ if (args.length == 0) {
+ usageMessage();
+ System.out.println("At least one argument specifying a workload is required.");
+ System.exit(0);
+ }
+
+ while (args[argindex].startsWith("-")) {
+ if (args[argindex].compareTo("-threads") == 0) {
+ argindex++;
+ if (argindex >= args.length) {
+ usageMessage();
+ System.out.println("Missing argument value for -threads.");
+ System.exit(0);
+ }
+ int tcount = Integer.parseInt(args[argindex]);
+ props.setProperty(THREAD_COUNT_PROPERTY, String.valueOf(tcount));
+ argindex++;
+ } else if (args[argindex].compareTo("-target") == 0) {
+ argindex++;
+ if (argindex >= args.length) {
+ usageMessage();
+ System.out.println("Missing argument value for -target.");
+ System.exit(0);
+ }
+ int ttarget = Integer.parseInt(args[argindex]);
+ props.setProperty(TARGET_PROPERTY, String.valueOf(ttarget));
+ argindex++;
+ } else if (args[argindex].compareTo("-load") == 0) {
+ props.setProperty(DO_TRANSACTIONS_PROPERTY, String.valueOf(false));
+ argindex++;
+ } else if (args[argindex].compareTo("-t") == 0) {
+ props.setProperty(DO_TRANSACTIONS_PROPERTY, String.valueOf(true));
+ argindex++;
+ } else if (args[argindex].compareTo("-s") == 0) {
+ props.setProperty(STATUS_PROPERTY, String.valueOf(true));
+ argindex++;
+ } else if (args[argindex].compareTo("-db") == 0) {
+ argindex++;
+ if (argindex >= args.length) {
+ usageMessage();
+ System.out.println("Missing argument value for -db.");
+ System.exit(0);
+ }
+ props.setProperty(DB_PROPERTY, args[argindex]);
+ argindex++;
+ } else if (args[argindex].compareTo("-l") == 0) {
+ argindex++;
+ if (argindex >= args.length) {
+ usageMessage();
+ System.out.println("Missing argument value for -l.");
+ System.exit(0);
+ }
+ props.setProperty(LABEL_PROPERTY, args[argindex]);
+ argindex++;
+ } else if (args[argindex].compareTo("-P") == 0) {
+ argindex++;
+ if (argindex >= args.length) {
+ usageMessage();
+ System.out.println("Missing argument value for -P.");
+ System.exit(0);
+ }
+ String propfile = args[argindex];
+ argindex++;
+
+ Properties myfileprops = new Properties();
+ try {
+ myfileprops.load(new FileInputStream(propfile));
+ } catch (IOException e) {
+ System.out.println("Unable to open the properties file " + propfile);
+ System.out.println(e.getMessage());
+ System.exit(0);
+ }
+
+ //Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5
+ for (Enumeration e = myfileprops.propertyNames(); e.hasMoreElements();) {
+ String prop = (String) e.nextElement();
+
+ fileprops.setProperty(prop, myfileprops.getProperty(prop));
+ }
+
+ } else if (args[argindex].compareTo("-p") == 0) {
+ argindex++;
+ if (argindex >= args.length) {
+ usageMessage();
+ System.out.println("Missing argument value for -p");
+ System.exit(0);
+ }
+ int eq = args[argindex].indexOf('=');
+ if (eq < 0) {
+ usageMessage();
+ System.out.println("Argument '-p' expected to be in key=value format (e.g., -p operationcount=99999)");
+ System.exit(0);
+ }
+
+ String name = args[argindex].substring(0, eq);
+ String value = args[argindex].substring(eq + 1);
+ props.put(name, value);
+ argindex++;
+ } else {
+ usageMessage();
+ System.out.println("Unknown option " + args[argindex]);
+ System.exit(0);
+ }
+
+ if (argindex >= args.length) {
+ break;
+ }
+ }
+
+ if (argindex != args.length) {
+ usageMessage();
+ if (argindex < args.length) {
+ System.out.println("An argument value without corresponding argument specifier (e.g., -p, -s) was found. "
+ + "We expected an argument specifier and instead found " + args[argindex]);
+ } else {
+ System.out.println("An argument specifier without corresponding value was found at the end of the supplied " +
+ "command line arguments.");
+ }
+ System.exit(0);
+ }
+
+ //overwrite file properties with properties from the command line
+
+ //Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5
+ for (Enumeration e = props.propertyNames(); e.hasMoreElements();) {
+ String prop = (String) e.nextElement();
+
+ fileprops.setProperty(prop, props.getProperty(prop));
+ }
+
+ props = fileprops;
+
+ if (!checkRequiredProperties(props)) {
+ System.out.println("Failed check required properties.");
+ System.exit(0);
+ }
+
+ return props;
+ }
+}
diff --git a/core/src/main/java/site/ycsb/ClientThread.java b/core/src/main/java/site/ycsb/ClientThread.java
new file mode 100644
index 0000000..50841a0
--- /dev/null
+++ b/core/src/main/java/site/ycsb/ClientThread.java
@@ -0,0 +1,187 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ * Copyright (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import site.ycsb.measurements.Measurements;
+import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.locks.LockSupport;
+
+/**
+ * A thread for executing transactions or data inserts to the database.
+ */
+public class ClientThread implements Runnable {
+ // Counts down each of the clients completing.
+ private final CountDownLatch completeLatch;
+
+ private static boolean spinSleep;
+ private DB db;
+ private boolean dotransactions;
+ private Workload workload;
+ private long opcount;
+ private double targetOpsPerMs;
+
+ private long opsdone;
+ private int threadid;
+ private int threadcount;
+ private Object workloadstate;
+ private Properties props;
+ private long targetOpsTickNs;
+ private final Measurements measurements;
+
+ /**
+ * Constructor.
+ *
+ * @param db the DB implementation to use
+ * @param dotransactions true to do transactions, false to insert data
+ * @param workload the workload to use
+ * @param props the properties defining the experiment
+ * @param opcount the number of operations (transactions or inserts) to do
+ * @param targetperthreadperms target number of operations per thread per ms
+ * @param completeLatch The latch tracking the completion of all clients.
+ */
+ public ClientThread(DB db, boolean dotransactions, Workload workload, Properties props, long opcount,
+ double targetperthreadperms, CountDownLatch completeLatch) {
+ this.db = db;
+ this.dotransactions = dotransactions;
+ this.workload = workload;
+ this.opcount = opcount;
+ opsdone = 0;
+ if (targetperthreadperms > 0) {
+ targetOpsPerMs = targetperthreadperms;
+ targetOpsTickNs = (long) (1000000 / targetOpsPerMs);
+ }
+ this.props = props;
+ measurements = Measurements.getMeasurements();
+ spinSleep = Boolean.valueOf(this.props.getProperty("spin.sleep", "false"));
+ this.completeLatch = completeLatch;
+ }
+
+ public void setThreadId(final int threadId) {
+ threadid = threadId;
+ }
+
+ public void setThreadCount(final int threadCount) {
+ threadcount = threadCount;
+ }
+
+ public long getOpsDone() {
+ return opsdone;
+ }
+
+ @Override
+ public void run() {
+ try {
+ db.init();
+ } catch (DBException e) {
+ e.printStackTrace();
+ e.printStackTrace(System.out);
+ return;
+ }
+
+ try {
+ workloadstate = workload.initThread(props, threadid, threadcount);
+ } catch (WorkloadException e) {
+ e.printStackTrace();
+ e.printStackTrace(System.out);
+ return;
+ }
+
+ //NOTE: Switching to using nanoTime and parkNanos for time management here such that the measurements
+ // and the client thread have the same view on time.
+
+ //spread the thread operations out so they don't all hit the DB at the same time
+ // GH issue 4 - throws exception if _target>1 because random.nextInt argument must be >0
+ // and the sleep() doesn't make sense for granularities < 1 ms anyway
+ if ((targetOpsPerMs > 0) && (targetOpsPerMs <= 1.0)) {
+ long randomMinorDelay = ThreadLocalRandom.current().nextInt((int) targetOpsTickNs);
+ sleepUntil(System.nanoTime() + randomMinorDelay);
+ }
+ try {
+ if (dotransactions) {
+ long startTimeNanos = System.nanoTime();
+
+ while (((opcount == 0) || (opsdone < opcount)) && !workload.isStopRequested()) {
+
+ if (!workload.doTransaction(db, workloadstate)) {
+ break;
+ }
+
+ opsdone++;
+
+ throttleNanos(startTimeNanos);
+ }
+ } else {
+ long startTimeNanos = System.nanoTime();
+
+ while (((opcount == 0) || (opsdone < opcount)) && !workload.isStopRequested()) {
+
+ if (!workload.doInsert(db, workloadstate)) {
+ break;
+ }
+
+ opsdone++;
+
+ throttleNanos(startTimeNanos);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ e.printStackTrace(System.out);
+ System.exit(0);
+ }
+
+ try {
+ measurements.setIntendedStartTimeNs(0);
+ db.cleanup();
+ } catch (DBException e) {
+ e.printStackTrace();
+ e.printStackTrace(System.out);
+ } finally {
+ completeLatch.countDown();
+ }
+ }
+
+ private static void sleepUntil(long deadline) {
+ while (System.nanoTime() < deadline) {
+ if (!spinSleep) {
+ LockSupport.parkNanos(deadline - System.nanoTime());
+ }
+ }
+ }
+
+ private void throttleNanos(long startTimeNanos) {
+ //throttle the operations
+ if (targetOpsPerMs > 0) {
+ // delay until next tick
+ long deadline = startTimeNanos + opsdone * targetOpsTickNs;
+ sleepUntil(deadline);
+ measurements.setIntendedStartTimeNs(deadline);
+ }
+ }
+
+ /**
+ * The total amount of work this thread is still expected to do.
+ */
+ long getOpsTodo() {
+ long todo = opcount - opsdone;
+ return todo < 0 ? 0 : todo;
+ }
+}
diff --git a/core/src/main/java/site/ycsb/CommandLine.java b/core/src/main/java/site/ycsb/CommandLine.java
new file mode 100644
index 0000000..a87cc4d
--- /dev/null
+++ b/core/src/main/java/site/ycsb/CommandLine.java
@@ -0,0 +1,354 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ * Copyright (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import site.ycsb.workloads.CoreWorkload;
+import site.ycsb.workloads.core.CoreConstants;
+import site.ycsb.wrappers.ByteIteratorWrapper;
+import site.ycsb.wrappers.DatabaseField;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.*;
+
+/**
+ * A simple command line client to a database, using the appropriate site.ycsb.DB implementation.
+ */
+public final class CommandLine {
+ private CommandLine() {
+ //not used
+ }
+
+ public static final String DEFAULT_DB = "site.ycsb.BasicDB";
+
+ public static void usageMessage() {
+ System.out.println("YCSB Command Line Client");
+ System.out.println("Usage: java site.ycsb.CommandLine [options]");
+ System.out.println("Options:");
+ System.out.println(" -P filename: Specify a property file");
+ System.out.println(" -p name=value: Specify a property value");
+ System.out.println(" -db classname: Use a specified DB class (can also set the \"db\" property)");
+ System.out.println(" -table tablename: Use the table name instead of the default \"" +
+ CoreConstants.TABLENAME_PROPERTY_DEFAULT + "\"");
+ System.out.println();
+ }
+
+ public static void help() {
+ System.out.println("Commands:");
+ System.out.println(" read key [field1 field2 ...] - Read a record");
+ System.out.println(" scan key recordcount [field1 field2 ...] - Scan starting at key");
+ System.out.println(" insert key name1=value1 [name2=value2 ...] - Insert a new record");
+ System.out.println(" update key name1=value1 [name2=value2 ...] - Update a record");
+ System.out.println(" delete key - Delete a record");
+ System.out.println(" table [tablename] - Get or [set] the name of the table");
+ System.out.println(" quit - Quit");
+ }
+
+ public static void main(String[] args) {
+
+ Properties props = new Properties();
+ Properties fileprops = new Properties();
+
+ parseArguments(args, props, fileprops);
+
+ for (Enumeration e = props.propertyNames(); e.hasMoreElements();) {
+ String prop = (String) e.nextElement();
+
+ fileprops.setProperty(prop, props.getProperty(prop));
+ }
+
+ props = fileprops;
+
+ System.out.println("YCSB Command Line client");
+ System.out.println("Type \"help\" for command line help");
+ System.out.println("Start with \"-help\" for usage info");
+
+ String table = props.getProperty(CoreConstants.TABLENAME_PROPERTY, CoreConstants.TABLENAME_PROPERTY_DEFAULT);
+
+ //create a DB
+ String dbname = props.getProperty(Client.DB_PROPERTY, DEFAULT_DB);
+
+ ClassLoader classLoader = CommandLine.class.getClassLoader();
+
+ DB db = null;
+
+ try {
+ Class dbclass = classLoader.loadClass(dbname);
+ db = (DB) dbclass.newInstance();
+ } catch (Exception e) {
+ e.printStackTrace();
+ System.exit(0);
+ }
+
+ db.setProperties(props);
+ try {
+ db.init();
+ } catch (DBException e) {
+ e.printStackTrace();
+ System.exit(0);
+ }
+
+ System.out.println("Connected.");
+
+ //main loop
+ BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+
+ for (;;) {
+ //get user input
+ System.out.print("> ");
+
+ String input = null;
+
+ try {
+ input = br.readLine();
+ } catch (IOException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ if (input.compareTo("") == 0) {
+ continue;
+ }
+
+ if (input.compareTo("help") == 0) {
+ help();
+ continue;
+ }
+
+ if (input.compareTo("quit") == 0) {
+ break;
+ }
+
+ String[] tokens = input.split(" ");
+
+ long st = System.currentTimeMillis();
+ //handle commands
+ if (tokens[0].compareTo("table") == 0) {
+ handleTable(tokens, table);
+ } else if (tokens[0].compareTo("read") == 0) {
+ handleRead(tokens, table, db);
+ } else if (tokens[0].compareTo("scan") == 0) {
+ handleScan(tokens, table, db);
+ } else if (tokens[0].compareTo("update") == 0) {
+ handleUpdate(tokens, table, db);
+ } else if (tokens[0].compareTo("insert") == 0) {
+ handleInsert(tokens, table, db);
+ } else if (tokens[0].compareTo("delete") == 0) {
+ handleDelete(tokens, table, db);
+ } else {
+ System.out.println("Error: unknown command \"" + tokens[0] + "\"");
+ }
+
+ System.out.println((System.currentTimeMillis() - st) + " ms");
+ }
+ }
+
+ private static void parseArguments(String[] args, Properties props, Properties fileprops) {
+ int argindex = 0;
+ while ((argindex < args.length) && (args[argindex].startsWith("-"))) {
+ if ((args[argindex].compareTo("-help") == 0) ||
+ (args[argindex].compareTo("--help") == 0) ||
+ (args[argindex].compareTo("-?") == 0) ||
+ (args[argindex].compareTo("--?") == 0)) {
+ usageMessage();
+ System.exit(0);
+ }
+
+ if (args[argindex].compareTo("-db") == 0) {
+ argindex++;
+ if (argindex >= args.length) {
+ usageMessage();
+ System.exit(0);
+ }
+ props.setProperty(Client.DB_PROPERTY, args[argindex]);
+ argindex++;
+ } else if (args[argindex].compareTo("-P") == 0) {
+ argindex++;
+ if (argindex >= args.length) {
+ usageMessage();
+ System.exit(0);
+ }
+ String propfile = args[argindex];
+ argindex++;
+
+ Properties myfileprops = new Properties();
+ try {
+ myfileprops.load(new FileInputStream(propfile));
+ } catch (IOException e) {
+ System.out.println(e.getMessage());
+ System.exit(0);
+ }
+
+ for (Enumeration e = myfileprops.propertyNames(); e.hasMoreElements();) {
+ String prop = (String) e.nextElement();
+
+ fileprops.setProperty(prop, myfileprops.getProperty(prop));
+ }
+
+ } else if (args[argindex].compareTo("-p") == 0) {
+ argindex++;
+ if (argindex >= args.length) {
+ usageMessage();
+ System.exit(0);
+ }
+ int eq = args[argindex].indexOf('=');
+ if (eq < 0) {
+ usageMessage();
+ System.exit(0);
+ }
+
+ String name = args[argindex].substring(0, eq);
+ String value = args[argindex].substring(eq + 1);
+ props.put(name, value);
+ argindex++;
+ } else if (args[argindex].compareTo("-table") == 0) {
+ argindex++;
+ if (argindex >= args.length) {
+ usageMessage();
+ System.exit(0);
+ }
+ props.put(CoreConstants.TABLENAME_PROPERTY, args[argindex]);
+
+ argindex++;
+ } else {
+ System.out.println("Unknown option " + args[argindex]);
+ usageMessage();
+ System.exit(0);
+ }
+
+ if (argindex >= args.length) {
+ break;
+ }
+ }
+
+ if (argindex != args.length) {
+ usageMessage();
+ System.exit(0);
+ }
+ }
+
+ private static void handleDelete(String[] tokens, String table, DB db) {
+ if (tokens.length != 2) {
+ System.out.println("Error: syntax is \"delete keyname\"");
+ } else {
+ Status ret = db.delete(table, tokens[1]);
+ System.out.println("Return result: " + ret.getName());
+ }
+ }
+
+ private static void handleInsert(String[] tokens, String table, DB db) {
+ if (tokens.length < 3) {
+ System.out.println("Error: syntax is \"insert keyname name1=value1 [name2=value2 ...]\"");
+ } else {
+ List values = new ArrayList<>();
+ for (int i = 2; i < tokens.length; i++) {
+ String[] nv = tokens[i].split("=");
+ values.add(new DatabaseField(
+ nv[0], ByteIteratorWrapper.create(new StringByteIterator(nv[1])))
+ );
+ }
+
+ Status ret = db.insert(table, tokens[1], values);
+ System.out.println("Result: " + ret.getName());
+ }
+ }
+
+ private static void handleUpdate(String[] tokens, String table, DB db) {
+ if (tokens.length < 3) {
+ System.out.println("Error: syntax is \"update keyname name1=value1 [name2=value2 ...]\"");
+ } else {
+ HashMap values = new HashMap<>();
+
+ for (int i = 2; i < tokens.length; i++) {
+ String[] nv = tokens[i].split("=");
+ values.put(nv[0], new StringByteIterator(nv[1]));
+ }
+
+ Status ret = db.update(table, tokens[1], values);
+ System.out.println("Result: " + ret.getName());
+ }
+ }
+
+ private static void handleScan(String[] tokens, String table, DB db) {
+ if (tokens.length < 3) {
+ System.out.println("Error: syntax is \"scan keyname scanlength [field1 field2 ...]\"");
+ } else {
+ Set fields = null;
+
+ if (tokens.length > 3) {
+ fields = new HashSet<>();
+
+ fields.addAll(Arrays.asList(tokens).subList(3, tokens.length));
+ }
+
+ Vector> results = new Vector<>();
+ Status ret = db.scan(table, tokens[1], Integer.parseInt(tokens[2]), fields, results);
+ System.out.println("Result: " + ret.getName());
+ int record = 0;
+ if (results.isEmpty()) {
+ System.out.println("0 records");
+ } else {
+ System.out.println("--------------------------------");
+ }
+ for (Map result : results) {
+ System.out.println("Record " + (record++));
+ for (Map.Entry ent : result.entrySet()) {
+ System.out.println(ent.getKey() + "=" + ent.getValue());
+ }
+ System.out.println("--------------------------------");
+ }
+ }
+ }
+
+ private static void handleRead(String[] tokens, String table, DB db) {
+ if (tokens.length == 1) {
+ System.out.println("Error: syntax is \"read keyname [field1 field2 ...]\"");
+ } else {
+ Set fields = null;
+
+ if (tokens.length > 2) {
+ fields = new HashSet<>();
+
+ fields.addAll(Arrays.asList(tokens).subList(2, tokens.length));
+ }
+
+ HashMap result = new HashMap<>();
+ Status ret = db.read(table, tokens[1], fields, result);
+ System.out.println("Return code: " + ret.getName());
+ for (Map.Entry ent : result.entrySet()) {
+ System.out.println(ent.getKey() + "=" + ent.getValue());
+ }
+ }
+ }
+
+ private static void handleTable(String[] tokens, String table) {
+ if (tokens.length == 1) {
+ System.out.println("Using table \"" + table + "\"");
+ } else if (tokens.length == 2) {
+ table = tokens[1];
+ System.out.println("Using table \"" + table + "\"");
+ } else {
+ System.out.println("Error: syntax is \"table tablename\"");
+ }
+ }
+
+
+}
diff --git a/core/src/main/java/site/ycsb/DB.java b/core/src/main/java/site/ycsb/DB.java
new file mode 100644
index 0000000..5b60578
--- /dev/null
+++ b/core/src/main/java/site/ycsb/DB.java
@@ -0,0 +1,148 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ * Copyright (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
+
+import site.ycsb.wrappers.DatabaseField;
+
+/**
+ * A layer for accessing a database to be benchmarked. Each thread in the client
+ * will be given its own instance of whatever DB class is to be used in the test.
+ * This class should be constructed using a no-argument constructor, so we can
+ * load it dynamically. Any argument-based initialization should be
+ * done by init().
+ *
+ * Note that YCSB does not make any use of the return codes returned by this class.
+ * Instead, it keeps a count of the return values and presents them to the user.
+ *
+ * The semantics of methods such as insert, update and delete vary from database
+ * to database. In particular, operations may or may not be durable once these
+ * methods commit, and some systems may return 'success' regardless of whether
+ * or not a tuple with a matching key existed before the call. Rather than dictate
+ * the exact semantics of these methods, we recommend you either implement them
+ * to match the database's default semantics, or the semantics of your
+ * target application. For the sake of comparison between experiments we also
+ * recommend you explain the semantics you chose when presenting performance results.
+ */
+public abstract class DB {
+ /**
+ * Properties for configuring this DB.
+ */
+ private Properties properties = new Properties();
+
+ /**
+ * Set the properties for this DB.
+ */
+ public void setProperties(Properties p) {
+ properties = p;
+
+ }
+
+ /**
+ * Get the set of properties for this DB.
+ */
+ public Properties getProperties() {
+ return properties;
+ }
+
+ /**
+ * Initialize any state for this DB.
+ * Called once per DB instance; there is one DB instance per client thread.
+ */
+ public void init() throws DBException {
+ }
+
+ /**
+ * Cleanup any state for this DB.
+ * Called once per DB instance; there is one DB instance per client thread.
+ */
+ public void cleanup() throws DBException {
+ }
+
+ /**
+ * Read a record from the database. Each field/value pair from the result will be stored in a HashMap.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to read.
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A HashMap of field/value pairs for the result
+ * @return The result of the operation.
+ */
+ public abstract Status read(String table, String key, Set fields, Map result);
+
+ /**
+ * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored
+ * in a HashMap.
+ *
+ * @param table The name of the table
+ * @param startkey The record key of the first record to read.
+ * @param recordcount The number of records to read
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
+ * @return The result of the operation.
+ */
+ public abstract Status scan(String table, String startkey, int recordcount, Set fields,
+ Vector> result);
+
+ /**
+ * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the
+ * record with the specified record key, overwriting any existing values with the same field name.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to write.
+ * @param values A HashMap of field/value pairs to update in the record
+ * @return The result of the operation.
+ */
+ public abstract Status update(String table, String key, Map values);
+
+ /**
+ * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the
+ * record with the specified record key.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to insert.
+ * @param values A HashMap of field/value pairs to insert in the record
+ * @return The result of the operation.
+ */
+ // public abstract Status insert(String table, String key, Map values);
+ public abstract Status insert(String table, String key, List values);
+
+ /**
+ * Delete a record from the database.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to delete.
+ * @return The result of the operation.
+ */
+ public abstract Status delete(String table, String key);
+
+ public static Map fieldListAsIteratorMap(List list) {
+ Map values = new HashMap<>();
+ for(DatabaseField f : list) {
+ values.put(f.getFieldname(), f.getContent().asIterator());
+ }
+ return values;
+ }
+}
diff --git a/core/src/main/java/site/ycsb/DBException.java b/core/src/main/java/site/ycsb/DBException.java
new file mode 100644
index 0000000..09ce9ec
--- /dev/null
+++ b/core/src/main/java/site/ycsb/DBException.java
@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+/**
+ * Something bad happened while interacting with the database.
+ */
+public class DBException extends Exception {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 6646883591588721475L;
+
+ public DBException(String message) {
+ super(message);
+ }
+
+ public DBException() {
+ super();
+ }
+
+ public DBException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public DBException(Throwable cause) {
+ super(cause);
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/DBFactory.java b/core/src/main/java/site/ycsb/DBFactory.java
new file mode 100644
index 0000000..4358633
--- /dev/null
+++ b/core/src/main/java/site/ycsb/DBFactory.java
@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ * Copyright (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import org.apache.htrace.core.Tracer;
+
+import java.util.Properties;
+
+/**
+ * Creates a DB layer by dynamically classloading the specified DB class.
+ */
+public final class DBFactory {
+ private DBFactory() {
+ // not used
+ }
+
+ public static DB newDB(String dbname, Properties properties, final Tracer tracer) throws UnknownDBException {
+ ClassLoader classLoader = DBFactory.class.getClassLoader();
+
+ DB ret;
+
+ try {
+ Class dbclass = classLoader.loadClass(dbname);
+
+ ret = (DB) dbclass.newInstance();
+ } catch (Exception e) {
+ e.printStackTrace();
+ return null;
+ }
+
+ ret.setProperties(properties);
+
+ return DBWrapper.createWrapper(ret, tracer);
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/DBWrapper.java b/core/src/main/java/site/ycsb/DBWrapper.java
new file mode 100644
index 0000000..fca435a
--- /dev/null
+++ b/core/src/main/java/site/ycsb/DBWrapper.java
@@ -0,0 +1,297 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc., 2016-2020 YCSB contributors. All rights reserved.
+ * Copyright (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import site.ycsb.measurements.Measurements;
+import site.ycsb.wrappers.Comparison;
+import site.ycsb.wrappers.DatabaseField;
+
+import org.apache.htrace.core.TraceScope;
+import org.apache.htrace.core.Tracer;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Wrapper around a "real" DB that measures latencies and counts return codes.
+ * Also reports latency separately between OK and failed operations.
+ */
+public class DBWrapper extends DB {
+ protected final DB db;
+ protected final Measurements measurements;
+ protected final Tracer tracer;
+
+ private boolean reportLatencyForEachError = false;
+ private Set latencyTrackedErrors = new HashSet();
+
+ private static final String REPORT_LATENCY_FOR_EACH_ERROR_PROPERTY = "reportlatencyforeacherror";
+ private static final String REPORT_LATENCY_FOR_EACH_ERROR_PROPERTY_DEFAULT = "false";
+
+ private static final String LATENCY_TRACKED_ERRORS_PROPERTY = "latencytrackederrors";
+
+ private static final AtomicBoolean LOG_REPORT_CONFIG = new AtomicBoolean(false);
+
+ private final String scopeStringCleanup;
+ private final String scopeStringDelete;
+ private final String scopeStringInit;
+ private final String scopeStringInsert;
+ private final String scopeStringRead;
+ private final String scopeStringScan;
+ private final String scopeStringUpdate;
+
+ public static DBWrapper createWrapper(final DB db, final Tracer tracer) {
+ if(db instanceof IndexableDB) {
+ return new IndexableDbWrapper(db, tracer);
+ }
+ return new DBWrapper(db, tracer);
+ }
+
+ protected DBWrapper(final DB db, final Tracer tracer) {
+ this.db = db;
+ measurements = Measurements.getMeasurements();
+ this.tracer = tracer;
+ final String simple = db.getClass().getSimpleName();
+ scopeStringCleanup = simple + "#cleanup";
+ scopeStringDelete = simple + "#delete";
+ scopeStringInit = simple + "#init";
+ scopeStringInsert = simple + "#insert";
+ scopeStringRead = simple + "#read";
+ scopeStringScan = simple + "#scan";
+ scopeStringUpdate = simple + "#update";
+ }
+
+ /**
+ * Set the properties for this DB.
+ */
+ public void setProperties(Properties p) {
+ db.setProperties(p);
+ }
+
+ /**
+ * Get the set of properties for this DB.
+ */
+ public Properties getProperties() {
+ return db.getProperties();
+ }
+
+ /**
+ * Initialize any state for this DB.
+ * Called once per DB instance; there is one DB instance per client thread.
+ */
+ public void init() throws DBException {
+ try (final TraceScope span = tracer.newScope(scopeStringInit)) {
+ db.init();
+
+ this.reportLatencyForEachError = Boolean.parseBoolean(getProperties().
+ getProperty(REPORT_LATENCY_FOR_EACH_ERROR_PROPERTY,
+ REPORT_LATENCY_FOR_EACH_ERROR_PROPERTY_DEFAULT));
+
+ if (!reportLatencyForEachError) {
+ String latencyTrackedErrorsProperty = getProperties().getProperty(LATENCY_TRACKED_ERRORS_PROPERTY, null);
+ if (latencyTrackedErrorsProperty != null) {
+ this.latencyTrackedErrors = new HashSet(Arrays.asList(
+ latencyTrackedErrorsProperty.split(",")));
+ }
+ }
+
+ if (LOG_REPORT_CONFIG.compareAndSet(false, true)) {
+ System.err.println("DBWrapper: report latency for each error is " +
+ this.reportLatencyForEachError + " and specific error codes to track" +
+ " for latency are: " + this.latencyTrackedErrors.toString());
+ }
+ }
+ }
+
+ /**
+ * Cleanup any state for this DB.
+ * Called once per DB instance; there is one DB instance per client thread.
+ */
+ public void cleanup() throws DBException {
+ try (final TraceScope span = tracer.newScope(scopeStringCleanup)) {
+ long ist = measurements.getIntendedStartTimeNs();
+ long st = System.nanoTime();
+ db.cleanup();
+ long en = System.nanoTime();
+ measure("CLEANUP", Status.OK, ist, st, en);
+ }
+ }
+
+ /**
+ * Read a record from the database. Each field/value pair from the result
+ * will be stored in a HashMap.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to read.
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A HashMap of field/value pairs for the result
+ * @return The result of the operation.
+ */
+ public Status read(String table, String key, Set fields,
+ Map result) {
+ try (final TraceScope span = tracer.newScope(scopeStringRead)) {
+ long ist = measurements.getIntendedStartTimeNs();
+ long st = System.nanoTime();
+ Status res = db.read(table, key, fields, result);
+ long en = System.nanoTime();
+ measure("READ", res, ist, st, en);
+ measurements.reportStatus("READ", res);
+ return res;
+ }
+ }
+
+ /**
+ * Perform a range scan for a set of records in the database.
+ * Each field/value pair from the result will be stored in a HashMap.
+ *
+ * @param table The name of the table
+ * @param startkey The record key of the first record to read.
+ * @param recordcount The number of records to read
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
+ * @return The result of the operation.
+ */
+ public Status scan(String table, String startkey, int recordcount,
+ Set fields, Vector> result) {
+ try (final TraceScope span = tracer.newScope(scopeStringScan)) {
+ long ist = measurements.getIntendedStartTimeNs();
+ long st = System.nanoTime();
+ Status res = db.scan(table, startkey, recordcount, fields, result);
+ long en = System.nanoTime();
+ measure("SCAN", res, ist, st, en);
+ measurements.reportStatus("SCAN", res);
+ return res;
+ }
+ }
+
+ protected final void measure(String op, Status result, long intendedStartTimeNanos,
+ long startTimeNanos, long endTimeNanos) {
+ String measurementName = op;
+ if (result == null || !result.isOk()) {
+ if (this.reportLatencyForEachError ||
+ this.latencyTrackedErrors.contains(result.getName())) {
+ measurementName = op + "-" + result.getName();
+ } else {
+ measurementName = op + "-FAILED";
+ }
+ }
+ measurements.measure(measurementName,
+ (int) ((endTimeNanos - startTimeNanos) / 1000));
+ measurements.measureIntended(measurementName,
+ (int) ((endTimeNanos - intendedStartTimeNanos) / 1000));
+ }
+
+ /**
+ * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the
+ * record with the specified record key, overwriting any existing values with the same field name.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to write.
+ * @param values A HashMap of field/value pairs to update in the record
+ * @return The result of the operation.
+ */
+ public Status update(String table, String key,
+ Map values) {
+ try (final TraceScope span = tracer.newScope(scopeStringUpdate)) {
+ long ist = measurements.getIntendedStartTimeNs();
+ long st = System.nanoTime();
+ Status res = db.update(table, key, values);
+ long en = System.nanoTime();
+ measure("UPDATE", res, ist, st, en);
+ measurements.reportStatus("UPDATE", res);
+ return res;
+ }
+ }
+
+ /**
+ * Insert a record in the database. Any field/value pairs in the specified
+ * values HashMap will be written into the record with the specified
+ * record key.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to insert.
+ * @param values A HashMap of field/value pairs to insert in the record
+ * @return The result of the operation.
+ */
+ public Status insert(String table, String key, List values) {
+ try (final TraceScope span = tracer.newScope(scopeStringInsert)) {
+ long ist = measurements.getIntendedStartTimeNs();
+ long st = System.nanoTime();
+ Status res = db.insert(table, key, values);
+ long en = System.nanoTime();
+ measure("INSERT", res, ist, st, en);
+ measurements.reportStatus("INSERT", res);
+ return res;
+ }
+ }
+
+ /**
+ * Delete a record from the database.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to delete.
+ * @return The result of the operation.
+ */
+ public Status delete(String table, String key) {
+ try (final TraceScope span = tracer.newScope(scopeStringDelete)) {
+ long ist = measurements.getIntendedStartTimeNs();
+ long st = System.nanoTime();
+ Status res = db.delete(table, key);
+ long en = System.nanoTime();
+ measure("DELETE", res, ist, st, en);
+ measurements.reportStatus("DELETE", res);
+ return res;
+ }
+ }
+}
+
+final class IndexableDbWrapper extends DBWrapper implements IndexableDB {
+ private final String scopeStringFindOne;
+ private final String scopeStringUpdateOne;
+
+ IndexableDbWrapper(final DB db, final Tracer tracer) {
+ super(db, tracer);
+ final String simple = db.getClass().getSimpleName();
+ scopeStringFindOne = simple + "#findone";
+ scopeStringUpdateOne = simple + "#updateone";
+ }
+ @Override
+ public Status findOne(String table, List filters, Set fields, Map result) {
+ try (final TraceScope span = tracer.newScope(scopeStringFindOne)) {
+ long ist = measurements.getIntendedStartTimeNs();
+ long st = System.nanoTime();
+ Status res = ((IndexableDB) db).findOne(table, filters, fields, result);
+ long en = System.nanoTime();
+ measure("FINDONE", res, ist, st, en);
+ measurements.reportStatus("FINDONE", res);
+ return res;
+ }
+ }
+ @Override
+ public Status updateOne(String table, List filters, List fields) {
+ try (final TraceScope span = tracer.newScope(scopeStringUpdateOne)) {
+ long ist = measurements.getIntendedStartTimeNs();
+ long st = System.nanoTime();
+ Status res = ((IndexableDB) db).updateOne(table, filters, fields);
+ long en = System.nanoTime();
+ measure("UPDATEONE", res, ist, st, en);
+ measurements.reportStatus("UPDATEONE", res);
+ return res;
+ }
+ }
+}
\ No newline at end of file
diff --git a/core/src/main/java/site/ycsb/GoodBadUglyDB.java b/core/src/main/java/site/ycsb/GoodBadUglyDB.java
new file mode 100644
index 0000000..ba3f8fc
--- /dev/null
+++ b/core/src/main/java/site/ycsb/GoodBadUglyDB.java
@@ -0,0 +1,165 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ * Copyright (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.Vector;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.LockSupport;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import site.ycsb.wrappers.DatabaseField;
+
+import static java.util.concurrent.TimeUnit.MICROSECONDS;
+
+/**
+ * Basic DB that just prints out the requested operations, instead of doing them against a database.
+ */
+public class GoodBadUglyDB extends DB {
+ public static final String SIMULATE_DELAY = "gbudb.delays";
+ public static final String SIMULATE_DELAY_DEFAULT = "200,1000,10000,50000,100000";
+ private static final ReadWriteLock DB_ACCESS = new ReentrantReadWriteLock();
+ private long[] delays;
+
+ public GoodBadUglyDB() {
+ delays = new long[]{200, 1000, 10000, 50000, 200000};
+ }
+
+ private void delay() {
+ final Random random = ThreadLocalRandom.current();
+ double p = random.nextDouble();
+ int mod;
+ if (p < 0.9) {
+ mod = 0;
+ } else if (p < 0.99) {
+ mod = 1;
+ } else if (p < 0.9999) {
+ mod = 2;
+ } else {
+ mod = 3;
+ }
+ // this will make mod 3 pauses global
+ Lock lock = mod == 3 ? DB_ACCESS.writeLock() : DB_ACCESS.readLock();
+ if (mod == 3) {
+ System.out.println("OUCH");
+ }
+ lock.lock();
+ try {
+ final long baseDelayNs = MICROSECONDS.toNanos(delays[mod]);
+ final int delayRangeNs = (int) (MICROSECONDS.toNanos(delays[mod + 1]) - baseDelayNs);
+ final long delayNs = baseDelayNs + random.nextInt(delayRangeNs);
+ final long deadline = System.nanoTime() + delayNs;
+ do {
+ LockSupport.parkNanos(deadline - System.nanoTime());
+ } while (System.nanoTime() < deadline && !Thread.interrupted());
+ } finally {
+ lock.unlock();
+ }
+
+ }
+
+ /**
+ * Initialize any state for this DB. Called once per DB instance; there is one DB instance per client thread.
+ */
+ public void init() {
+ int i = 0;
+ for (String delay : getProperties().getProperty(SIMULATE_DELAY, SIMULATE_DELAY_DEFAULT).split(",")) {
+ delays[i++] = Long.parseLong(delay);
+ }
+ }
+
+ /**
+ * Read a record from the database. Each field/value pair from the result will be stored in a HashMap.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to read.
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A HashMap of field/value pairs for the result
+ * @return Zero on success, a non-zero error code on error
+ */
+ public Status read(String table, String key, Set fields, Map result) {
+ delay();
+ return Status.OK;
+ }
+
+ /**
+ * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored
+ * in a HashMap.
+ *
+ * @param table The name of the table
+ * @param startkey The record key of the first record to read.
+ * @param recordcount The number of records to read
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
+ * @return Zero on success, a non-zero error code on error
+ */
+ public Status scan(String table, String startkey, int recordcount, Set fields,
+ Vector> result) {
+ delay();
+
+ return Status.OK;
+ }
+
+ /**
+ * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the
+ * record with the specified record key, overwriting any existing values with the same field name.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to write.
+ * @param values A HashMap of field/value pairs to update in the record
+ * @return Zero on success, a non-zero error code on error
+ */
+ public Status update(String table, String key, Map values) {
+ delay();
+
+ return Status.OK;
+ }
+
+ /**
+ * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the
+ * record with the specified record key.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to insert.
+ * @param values A HashMap of field/value pairs to insert in the record
+ * @return Zero on success, a non-zero error code on error
+ */
+ public Status insert(String table, String key, List values) {
+ delay();
+ return Status.OK;
+ }
+
+ /**
+ * Delete a record from the database.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to delete.
+ * @return Zero on success, a non-zero error code on error
+ */
+ public Status delete(String table, String key) {
+ delay();
+ return Status.OK;
+ }
+}
diff --git a/core/src/main/java/site/ycsb/IndexableDB.java b/core/src/main/java/site/ycsb/IndexableDB.java
new file mode 100644
index 0000000..f650103
--- /dev/null
+++ b/core/src/main/java/site/ycsb/IndexableDB.java
@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) 2023-204 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import site.ycsb.wrappers.Comparison;
+import site.ycsb.wrappers.DatabaseField;
+
+public interface IndexableDB {
+
+ public static final String TYPED_FIELDS_PROPERTY = "typedfields";
+ public static final String TYPED_FIELDS_DEFAULT = "false";
+
+ public Status findOne(String table, List filters,
+ Set fields, Map result);
+ public Status updateOne(String table, List filters, List fields);
+}
diff --git a/core/src/main/java/site/ycsb/InputStreamByteIterator.java b/core/src/main/java/site/ycsb/InputStreamByteIterator.java
new file mode 100644
index 0000000..e803798
--- /dev/null
+++ b/core/src/main/java/site/ycsb/InputStreamByteIterator.java
@@ -0,0 +1,96 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * A ByteIterator that iterates through an inputstream of bytes.
+ */
+public class InputStreamByteIterator extends ByteIterator {
+ private final long len;
+ private final InputStream ins;
+ private long off;
+ private final boolean resetable;
+
+ public InputStreamByteIterator(InputStream ins, long len) {
+ this.len = len;
+ this.ins = ins;
+ off = 0;
+ resetable = ins.markSupported();
+ if (resetable) {
+ ins.mark((int) len);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return off < len;
+ }
+
+ @Override
+ public byte nextByte() {
+ int ret;
+ try {
+ ret = ins.read();
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ if (ret == -1) {
+ throw new IllegalStateException("Past EOF!");
+ }
+ off++;
+ return (byte) ret;
+ }
+
+ @Override
+ public long bytesLeft() {
+ return len - off;
+ }
+
+ @Override
+ public byte[] toArray() {
+ int size = (int) bytesLeft();
+ byte[] bytes = new byte[size];
+ try {
+ if (ins.read(bytes) < size) {
+ throw new IllegalStateException("Past EOF!");
+ }
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ off = len;
+ return bytes;
+ }
+
+ @Override
+ public void reset() {
+ if (resetable) {
+ try {
+ ins.reset();
+ ins.mark((int) len);
+ off = 0;
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to reset the input stream", e);
+ }
+ } else {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/NumericByteIterator.java b/core/src/main/java/site/ycsb/NumericByteIterator.java
new file mode 100644
index 0000000..42e82b0
--- /dev/null
+++ b/core/src/main/java/site/ycsb/NumericByteIterator.java
@@ -0,0 +1,79 @@
+/**
+ * Copyright (c) 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb;
+
+/**
+ * A byte iterator that handles encoding and decoding numeric values.
+ * Currently this iterator can handle 64 bit signed values and double precision
+ * floating point values.
+ */
+public class NumericByteIterator extends ByteIterator {
+ private final byte[] payload;
+ private final boolean floatingPoint;
+ private int off;
+
+ public NumericByteIterator(final long value) {
+ floatingPoint = false;
+ payload = Utils.longToBytes(value);
+ off = 0;
+ }
+
+ public NumericByteIterator(final double value) {
+ floatingPoint = true;
+ payload = Utils.doubleToBytes(value);
+ off = 0;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return off < payload.length;
+ }
+
+ @Override
+ public byte nextByte() {
+ return payload[off++];
+ }
+
+ @Override
+ public long bytesLeft() {
+ return payload.length - off;
+ }
+
+ @Override
+ public void reset() {
+ off = 0;
+ }
+
+ public long getLong() {
+ if (floatingPoint) {
+ throw new IllegalStateException("Byte iterator is of the type double");
+ }
+ return Utils.bytesToLong(payload);
+ }
+
+ public double getDouble() {
+ if (!floatingPoint) {
+ throw new IllegalStateException("Byte iterator is of the type long");
+ }
+ return Utils.bytesToDouble(payload);
+ }
+
+ public boolean isFloatingPoint() {
+ return floatingPoint;
+ }
+
+}
\ No newline at end of file
diff --git a/core/src/main/java/site/ycsb/RandomByteIterator.java b/core/src/main/java/site/ycsb/RandomByteIterator.java
new file mode 100644
index 0000000..4851c57
--- /dev/null
+++ b/core/src/main/java/site/ycsb/RandomByteIterator.java
@@ -0,0 +1,118 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * A ByteIterator that generates a random sequence of bytes.
+ */
+public class RandomByteIterator extends ByteIterator {
+ private final long len;
+ private long off;
+ private int bufOff;
+ private final byte[] buf;
+
+ @Override
+ public boolean hasNext() {
+ return (off + bufOff) < len;
+ }
+
+ private void fillBytesImpl(byte[] buffer, int base) {
+ int bytes = ThreadLocalRandom.current().nextInt();
+
+ switch (buffer.length - base) {
+ default:
+ buffer[base + 5] = (byte) (((bytes >> 25) & 95) + ' ');
+ case 5:
+ buffer[base + 4] = (byte) (((bytes >> 20) & 63) + ' ');
+ case 4:
+ buffer[base + 3] = (byte) (((bytes >> 15) & 31) + ' ');
+ case 3:
+ buffer[base + 2] = (byte) (((bytes >> 10) & 95) + ' ');
+ case 2:
+ buffer[base + 1] = (byte) (((bytes >> 5) & 63) + ' ');
+ case 1:
+ buffer[base + 0] = (byte) (((bytes) & 31) + ' ');
+ case 0:
+ break;
+ }
+ }
+
+ private void fillBytes() {
+ if (bufOff == buf.length) {
+ fillBytesImpl(buf, 0);
+ bufOff = 0;
+ off += buf.length;
+ }
+ }
+
+ public RandomByteIterator(long len) {
+ this.len = len;
+ this.buf = new byte[6];
+ this.bufOff = buf.length;
+ fillBytes();
+ this.off = 0;
+ }
+
+ public byte nextByte() {
+ fillBytes();
+ bufOff++;
+ return buf[bufOff - 1];
+ }
+
+ @Override
+ public int nextBuf(byte[] buffer, int bufOffset) {
+ int ret;
+ if (len - off < buffer.length - bufOffset) {
+ ret = (int) (len - off);
+ } else {
+ ret = buffer.length - bufOffset;
+ }
+ int i;
+ for (i = 0; i < ret; i += 6) {
+ fillBytesImpl(buffer, i + bufOffset);
+ }
+ off += ret;
+ return ret + bufOffset;
+ }
+
+ @Override
+ public long bytesLeft() {
+ return len - off - bufOff;
+ }
+
+ @Override
+ public void reset() {
+ off = 0;
+ }
+
+ /** Consumes remaining contents of this object, and returns them as a byte array. */
+ public byte[] toArray() {
+ long left = bytesLeft();
+ if (left != (int) left) {
+ throw new ArrayIndexOutOfBoundsException("Too much data to fit in one array!");
+ }
+ byte[] ret = new byte[(int) left];
+ int bufOffset = 0;
+ while (bufOffset < ret.length) {
+ bufOffset = nextBuf(ret, bufOffset);
+ }
+ return ret;
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/Status.java b/core/src/main/java/site/ycsb/Status.java
new file mode 100644
index 0000000..8a27cc9
--- /dev/null
+++ b/core/src/main/java/site/ycsb/Status.java
@@ -0,0 +1,110 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+/**
+ * The result of an operation.
+ */
+public class Status {
+ private final String name;
+ private final String description;
+
+ /**
+ * @param name A short name for the status.
+ * @param description A description of the status.
+ */
+ public Status(String name, String description) {
+ super();
+ this.name = name;
+ this.description = description;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ @Override
+ public String toString() {
+ return "Status [name=" + name + ", description=" + description + "]";
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((description == null) ? 0 : description.hashCode());
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ Status other = (Status) obj;
+ if (description == null) {
+ if (other.description != null) {
+ return false;
+ }
+ } else if (!description.equals(other.description)) {
+ return false;
+ }
+ if (name == null) {
+ if (other.name != null) {
+ return false;
+ }
+ } else if (!name.equals(other.name)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Is {@code this} a passing state for the operation: {@link Status#OK} or {@link Status#BATCHED_OK}.
+ * @return true if the operation is successful, false otherwise
+ */
+ public boolean isOk() {
+ return this == OK || this == BATCHED_OK;
+ }
+
+ public static final Status OK = new Status("OK", "The operation completed successfully.");
+ public static final Status ERROR = new Status("ERROR", "The operation failed.");
+ public static final Status NOT_FOUND = new Status("NOT_FOUND", "The requested record was not found.");
+ public static final Status NOT_IMPLEMENTED = new Status("NOT_IMPLEMENTED", "The operation is not " +
+ "implemented for the current binding.");
+ public static final Status UNEXPECTED_STATE = new Status("UNEXPECTED_STATE", "The operation reported" +
+ " success, but the result was not as expected.");
+ public static final Status BAD_REQUEST = new Status("BAD_REQUEST", "The request was not valid.");
+ public static final Status FORBIDDEN = new Status("FORBIDDEN", "The operation is forbidden.");
+ public static final Status SERVICE_UNAVAILABLE = new Status("SERVICE_UNAVAILABLE", "Dependant " +
+ "service for the current binding is not available.");
+ public static final Status BATCHED_OK = new Status("BATCHED_OK", "The operation has been batched by " +
+ "the binding to be executed later.");
+}
+
diff --git a/core/src/main/java/site/ycsb/StatusThread.java b/core/src/main/java/site/ycsb/StatusThread.java
new file mode 100644
index 0000000..a45caeb
--- /dev/null
+++ b/core/src/main/java/site/ycsb/StatusThread.java
@@ -0,0 +1,308 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import site.ycsb.measurements.Measurements;
+import java.text.DecimalFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A thread to periodically show the status of the experiment to reassure you that progress is being made.
+ */
+public class StatusThread extends Thread {
+ // Counts down each of the clients completing
+ private final CountDownLatch completeLatch;
+
+ // Stores the measurements for the run
+ private final Measurements measurements;
+
+ // Whether or not to track the JVM stats per run
+ private final boolean trackJVMStats;
+
+ // The clients that are running.
+ private final List clients;
+
+ private final String label;
+ private final boolean standardstatus;
+
+ // The interval for reporting status.
+ private long sleeptimeNs;
+
+ // JVM max/mins
+ private int maxThreads;
+ private int minThreads = Integer.MAX_VALUE;
+ private long maxUsedMem;
+ private long minUsedMem = Long.MAX_VALUE;
+ private double maxLoadAvg;
+ private double minLoadAvg = Double.MAX_VALUE;
+ private long lastGCCount = 0;
+ private long lastGCTime = 0;
+
+ /**
+ * Creates a new StatusThread without JVM stat tracking.
+ *
+ * @param completeLatch The latch that each client thread will {@link CountDownLatch#countDown()}
+ * as they complete.
+ * @param clients The clients to collect metrics from.
+ * @param label The label for the status.
+ * @param standardstatus If true the status is printed to stdout in addition to stderr.
+ * @param statusIntervalSeconds The number of seconds between status updates.
+ */
+ public StatusThread(CountDownLatch completeLatch, List clients,
+ String label, boolean standardstatus, int statusIntervalSeconds) {
+ this(completeLatch, clients, label, standardstatus, statusIntervalSeconds, false);
+ }
+
+ /**
+ * Creates a new StatusThread.
+ *
+ * @param completeLatch The latch that each client thread will {@link CountDownLatch#countDown()}
+ * as they complete.
+ * @param clients The clients to collect metrics from.
+ * @param label The label for the status.
+ * @param standardstatus If true the status is printed to stdout in addition to stderr.
+ * @param statusIntervalSeconds The number of seconds between status updates.
+ * @param trackJVMStats Whether or not to track JVM stats.
+ */
+ public StatusThread(CountDownLatch completeLatch, List clients,
+ String label, boolean standardstatus, int statusIntervalSeconds,
+ boolean trackJVMStats) {
+ this.completeLatch = completeLatch;
+ this.clients = clients;
+ this.label = label;
+ this.standardstatus = standardstatus;
+ sleeptimeNs = TimeUnit.SECONDS.toNanos(statusIntervalSeconds);
+ measurements = Measurements.getMeasurements();
+ this.trackJVMStats = trackJVMStats;
+ }
+
+ /**
+ * Run and periodically report status.
+ */
+ @Override
+ public void run() {
+ final long startTimeMs = System.currentTimeMillis();
+ final long startTimeNanos = System.nanoTime();
+ long deadline = startTimeNanos + sleeptimeNs;
+ long startIntervalMs = startTimeMs;
+ long lastTotalOps = 0;
+
+ boolean alldone;
+
+ do {
+ long nowMs = System.currentTimeMillis();
+
+ lastTotalOps = computeStats(startTimeMs, startIntervalMs, nowMs, lastTotalOps);
+
+ if (trackJVMStats) {
+ measureJVM();
+ }
+
+ alldone = waitForClientsUntil(deadline);
+
+ startIntervalMs = nowMs;
+ deadline += sleeptimeNs;
+ }
+ while (!alldone);
+
+ if (trackJVMStats) {
+ measureJVM();
+ }
+ // Print the final stats.
+ computeStats(startTimeMs, startIntervalMs, System.currentTimeMillis(), lastTotalOps);
+ }
+
+ /**
+ * Computes and prints the stats.
+ *
+ * @param startTimeMs The start time of the test.
+ * @param startIntervalMs The start time of this interval.
+ * @param endIntervalMs The end time (now) for the interval.
+ * @param lastTotalOps The last total operations count.
+ * @return The current operation count.
+ */
+ private long computeStats(final long startTimeMs, long startIntervalMs, long endIntervalMs,
+ long lastTotalOps) {
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS");
+
+ long totalops = 0;
+ long todoops = 0;
+
+ // Calculate the total number of operations completed.
+ for (ClientThread t : clients) {
+ totalops += t.getOpsDone();
+ todoops += t.getOpsTodo();
+ }
+
+
+ long interval = endIntervalMs - startTimeMs;
+ double throughput = 1000.0 * (((double) totalops) / (double) interval);
+ double curthroughput = 1000.0 * (((double) (totalops - lastTotalOps)) /
+ ((double) (endIntervalMs - startIntervalMs)));
+ long estremaining = (long) Math.ceil(todoops / throughput);
+
+
+ DecimalFormat d = new DecimalFormat("#.##");
+ String labelString = this.label + format.format(new Date());
+
+ StringBuilder msg = new StringBuilder(labelString).append(" ").append(interval / 1000).append(" sec: ");
+ msg.append(totalops).append(" operations; ");
+
+ if (totalops != 0) {
+ msg.append(d.format(curthroughput)).append(" current ops/sec; ");
+ }
+ if (todoops != 0) {
+ msg.append("est completion in ").append(RemainingFormatter.format(estremaining));
+ }
+
+ msg.append(Measurements.getMeasurements().getSummary());
+
+ System.err.println(msg);
+
+ if (standardstatus) {
+ System.out.println(msg);
+ }
+ return totalops;
+ }
+
+ /**
+ * Waits for all of the client to finish or the deadline to expire.
+ *
+ * @param deadline The current deadline.
+ * @return True if all of the clients completed.
+ */
+ private boolean waitForClientsUntil(long deadline) {
+ boolean alldone = false;
+ long now = System.nanoTime();
+
+ while (!alldone && now < deadline) {
+ try {
+ alldone = completeLatch.await(deadline - now, TimeUnit.NANOSECONDS);
+ } catch (InterruptedException ie) {
+ // If we are interrupted the thread is being asked to shutdown.
+ // Return true to indicate that and reset the interrupt state
+ // of the thread.
+ Thread.currentThread().interrupt();
+ alldone = true;
+ }
+ now = System.nanoTime();
+ }
+
+ return alldone;
+ }
+
+ /**
+ * Executes the JVM measurements.
+ */
+ private void measureJVM() {
+ final int threads = Utils.getActiveThreadCount();
+ if (threads < minThreads) {
+ minThreads = threads;
+ }
+ if (threads > maxThreads) {
+ maxThreads = threads;
+ }
+ measurements.measure("THREAD_COUNT", threads);
+
+ // TODO - once measurements allow for other number types, switch to using
+ // the raw bytes. Otherwise we can track in MB to avoid negative values
+ // when faced with huge heaps.
+ final int usedMem = Utils.getUsedMemoryMegaBytes();
+ if (usedMem < minUsedMem) {
+ minUsedMem = usedMem;
+ }
+ if (usedMem > maxUsedMem) {
+ maxUsedMem = usedMem;
+ }
+ measurements.measure("USED_MEM_MB", usedMem);
+
+ // Some JVMs may not implement this feature so if the value is less than
+ // zero, just ommit it.
+ final double systemLoad = Utils.getSystemLoadAverage();
+ if (systemLoad >= 0) {
+ // TODO - store the double if measurements allows for them
+ measurements.measure("SYS_LOAD_AVG", (int) systemLoad);
+ if (systemLoad > maxLoadAvg) {
+ maxLoadAvg = systemLoad;
+ }
+ if (systemLoad < minLoadAvg) {
+ minLoadAvg = systemLoad;
+ }
+ }
+
+ final long gcs = Utils.getGCTotalCollectionCount();
+ measurements.measure("GCS", (int) (gcs - lastGCCount));
+ final long gcTime = Utils.getGCTotalTime();
+ measurements.measure("GCS_TIME", (int) (gcTime - lastGCTime));
+ lastGCCount = gcs;
+ lastGCTime = gcTime;
+ }
+
+ /**
+ * @return The maximum threads running during the test.
+ */
+ public int getMaxThreads() {
+ return maxThreads;
+ }
+
+ /**
+ * @return The minimum threads running during the test.
+ */
+ public int getMinThreads() {
+ return minThreads;
+ }
+
+ /**
+ * @return The maximum memory used during the test.
+ */
+ public long getMaxUsedMem() {
+ return maxUsedMem;
+ }
+
+ /**
+ * @return The minimum memory used during the test.
+ */
+ public long getMinUsedMem() {
+ return minUsedMem;
+ }
+
+ /**
+ * @return The maximum load average during the test.
+ */
+ public double getMaxLoadAvg() {
+ return maxLoadAvg;
+ }
+
+ /**
+ * @return The minimum load average during the test.
+ */
+ public double getMinLoadAvg() {
+ return minLoadAvg;
+ }
+
+ /**
+ * @return Whether or not the thread is tracking JVM stats.
+ */
+ public boolean trackJVMStats() {
+ return trackJVMStats;
+ }
+}
diff --git a/core/src/main/java/site/ycsb/StringByteIterator.java b/core/src/main/java/site/ycsb/StringByteIterator.java
new file mode 100644
index 0000000..5c899ce
--- /dev/null
+++ b/core/src/main/java/site/ycsb/StringByteIterator.java
@@ -0,0 +1,131 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A ByteIterator that iterates through a string.
+ */
+public class StringByteIterator extends ByteIterator {
+ private String str;
+ private int off;
+
+ /**
+ * Put all of the entries of one map into the other, converting
+ * String values into ByteIterators.
+ */
+ public static void putAllAsByteIterators(Map out, Map in) {
+ for (Map.Entry entry : in.entrySet()) {
+ out.put(entry.getKey(), new StringByteIterator(entry.getValue()));
+ }
+ }
+
+ /**
+ * Put all of the entries of one map into the other, converting
+ * ByteIterator values into Strings.
+ */
+ public static void putAllAsStrings(Map out, Map in) {
+ for (Map.Entry entry : in.entrySet()) {
+ out.put(entry.getKey(), entry.getValue().toString());
+ }
+ }
+
+ /**
+ * Create a copy of a map, converting the values from Strings to
+ * StringByteIterators.
+ */
+ public static Map getByteIteratorMap(Map m) {
+ HashMap ret =
+ new HashMap();
+
+ for (Map.Entry entry : m.entrySet()) {
+ ret.put(entry.getKey(), new StringByteIterator(entry.getValue()));
+ }
+ return ret;
+ }
+
+ /**
+ * Create a copy of a map, converting the values from
+ * StringByteIterators to Strings.
+ */
+ public static Map getStringMap(Map m) {
+ HashMap ret = new HashMap();
+
+ for (Map.Entry entry : m.entrySet()) {
+ ret.put(entry.getKey(), entry.getValue().toString());
+ }
+ return ret;
+ }
+
+ public StringByteIterator(String s) {
+ this.str = s;
+ this.off = 0;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return off < str.length();
+ }
+
+ @Override
+ public byte nextByte() {
+ byte ret = (byte) str.charAt(off);
+ off++;
+ return ret;
+ }
+
+ @Override
+ public long bytesLeft() {
+ return str.length() - off;
+ }
+
+ @Override
+ public void reset() {
+ off = 0;
+ }
+
+ @Override
+ public byte[] toArray() {
+ byte[] bytes = new byte[(int) bytesLeft()];
+ for (int i = 0; i < bytes.length; i++) {
+ bytes[i] = (byte) str.charAt(off + i);
+ }
+ off = str.length();
+ return bytes;
+ }
+
+ /**
+ * Specialization of general purpose toString() to avoid unnecessary
+ * copies.
+ *
+ * Creating a new StringByteIterator, then calling toString()
+ * yields the original String object, and does not perform any copies
+ * or String conversion operations.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb;
+
+import java.util.Collection;
+
+/**
+ * A thread that waits for the maximum specified time and then interrupts all the client
+ * threads passed at initialization of this thread.
+ *
+ * The maximum execution time passed is assumed to be in seconds.
+ *
+ */
+public class TerminatorThread extends Thread {
+
+ private final Collection threads;
+ private long maxExecutionTime;
+ private Workload workload;
+ private long waitTimeOutInMS;
+
+ public TerminatorThread(long maxExecutionTime, Collection threads,
+ Workload workload) {
+ this.maxExecutionTime = maxExecutionTime;
+ this.threads = threads;
+ this.workload = workload;
+ waitTimeOutInMS = 2000;
+ System.err.println("Maximum execution time specified as: " + maxExecutionTime + " secs");
+ }
+
+ public void run() {
+ try {
+ Thread.sleep(maxExecutionTime * 1000);
+ } catch (InterruptedException e) {
+ System.err.println("Could not wait until max specified time, TerminatorThread interrupted.");
+ return;
+ }
+ System.err.println("Maximum time elapsed. Requesting stop for the workload.");
+ workload.requestStop();
+ System.err.println("Stop requested for workload. Now Joining!");
+ for (Thread t : threads) {
+ while (t.isAlive()) {
+ try {
+ t.join(waitTimeOutInMS);
+ if (t.isAlive()) {
+ System.out.println("Still waiting for thread " + t.getName() + " to complete. " +
+ "Workload status: " + workload.isStopRequested());
+ }
+ } catch (InterruptedException e) {
+ // Do nothing. Don't know why I was interrupted.
+ }
+ }
+ }
+ }
+}
diff --git a/core/src/main/java/site/ycsb/TimeseriesDB.java b/core/src/main/java/site/ycsb/TimeseriesDB.java
new file mode 100644
index 0000000..eda6997
--- /dev/null
+++ b/core/src/main/java/site/ycsb/TimeseriesDB.java
@@ -0,0 +1,351 @@
+/*
+ * Copyright (c) 2018 YCSB Contributors All rights reserved.
+ * Copyright (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb;
+
+import site.ycsb.generator.Generator;
+import site.ycsb.generator.IncrementingPrintableStringGenerator;
+import site.ycsb.workloads.TimeSeriesWorkload;
+import site.ycsb.wrappers.DatabaseField;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Abstract class to adapt the default ycsb DB interface to Timeseries databases.
+ * This class is mostly here to be extended by Timeseries dataabases
+ * originally developed by Andreas Bader in YCSB-TS.
+ *
+ * This class is mostly parsing the workload information passed through the default ycsb interface
+ * according to the information outlined in {@link TimeSeriesWorkload}.
+ * It also contains some minor utility methods relevant to Timeseries databases.
+ *
+ *
+ * @implSpec It's vital to call super.init() when overwriting the init method
+ * to correctly initialize the workload-parsing.
+ */
+public abstract class TimeseriesDB extends DB {
+
+ // defaults for downsampling. Basically we ignore it
+ private static final String DOWNSAMPLING_FUNCTION_PROPERTY_DEFAULT = "NONE";
+ private static final String DOWNSAMPLING_INTERVAL_PROPERTY_DEFAULT = "0";
+
+ // debug property loading
+ private static final String DEBUG_PROPERTY = "debug";
+ private static final String DEBUG_PROPERTY_DEFAULT = "false";
+
+ // test property loading
+ private static final String TEST_PROPERTY = "test";
+ private static final String TEST_PROPERTY_DEFAULT = "false";
+
+ // Workload parameters that we need to parse this
+ protected String timestampKey;
+ protected String valueKey;
+ protected String tagPairDelimiter;
+ protected String queryTimeSpanDelimiter;
+ protected String deleteDelimiter;
+ protected TimeUnit timestampUnit;
+ protected String groupByKey;
+ protected String downsamplingKey;
+ protected Integer downsamplingInterval;
+ protected AggregationOperation downsamplingFunction;
+
+ // YCSB-parameters
+ protected boolean debug;
+ protected boolean test;
+
+ /**
+ * Initialize any state for this DB.
+ * Called once per DB instance; there is one DB instance per client thread.
+ */
+ @Override
+ public void init() throws DBException {
+ // taken from BasicTSDB
+ timestampKey = getProperties().getProperty(
+ TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY,
+ TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT);
+ valueKey = getProperties().getProperty(
+ TimeSeriesWorkload.VALUE_KEY_PROPERTY,
+ TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT);
+ tagPairDelimiter = getProperties().getProperty(
+ TimeSeriesWorkload.PAIR_DELIMITER_PROPERTY,
+ TimeSeriesWorkload.PAIR_DELIMITER_PROPERTY_DEFAULT);
+ queryTimeSpanDelimiter = getProperties().getProperty(
+ TimeSeriesWorkload.QUERY_TIMESPAN_DELIMITER_PROPERTY,
+ TimeSeriesWorkload.QUERY_TIMESPAN_DELIMITER_PROPERTY_DEFAULT);
+ deleteDelimiter = getProperties().getProperty(
+ TimeSeriesWorkload.DELETE_DELIMITER_PROPERTY,
+ TimeSeriesWorkload.DELETE_DELIMITER_PROPERTY_DEFAULT);
+ timestampUnit = TimeUnit.valueOf(getProperties().getProperty(
+ TimeSeriesWorkload.TIMESTAMP_UNITS_PROPERTY,
+ TimeSeriesWorkload.TIMESTAMP_UNITS_PROPERTY_DEFAULT));
+ groupByKey = getProperties().getProperty(
+ TimeSeriesWorkload.GROUPBY_KEY_PROPERTY,
+ TimeSeriesWorkload.GROUPBY_KEY_PROPERTY_DEFAULT);
+ downsamplingKey = getProperties().getProperty(
+ TimeSeriesWorkload.DOWNSAMPLING_KEY_PROPERTY,
+ TimeSeriesWorkload.DOWNSAMPLING_KEY_PROPERTY_DEFAULT);
+ downsamplingFunction = TimeseriesDB.AggregationOperation.valueOf(getProperties()
+ .getProperty(TimeSeriesWorkload.DOWNSAMPLING_FUNCTION_PROPERTY, DOWNSAMPLING_FUNCTION_PROPERTY_DEFAULT));
+ downsamplingInterval = Integer.valueOf(getProperties()
+ .getProperty(TimeSeriesWorkload.DOWNSAMPLING_INTERVAL_PROPERTY, DOWNSAMPLING_INTERVAL_PROPERTY_DEFAULT));
+
+ test = Boolean.parseBoolean(getProperties().getProperty(TEST_PROPERTY, TEST_PROPERTY_DEFAULT));
+ debug = Boolean.parseBoolean(getProperties().getProperty(DEBUG_PROPERTY, DEBUG_PROPERTY_DEFAULT));
+ }
+
+ @Override
+ public final Status read(String table, String key, Set fields, Map result) {
+ Map> tagQueries = new HashMap<>();
+ Long timestamp = null;
+ for (String field : fields) {
+ if (field.startsWith(timestampKey)) {
+ String[] timestampParts = field.split(tagPairDelimiter);
+ if (timestampParts[1].contains(queryTimeSpanDelimiter)) {
+ // Since we're looking for a single datapoint, a range of timestamps makes no sense.
+ // As we cannot throw an exception to bail out here, we return `BAD_REQUEST` instead.
+ return Status.BAD_REQUEST;
+ }
+ timestamp = Long.valueOf(timestampParts[1]);
+ } else {
+ String[] queryParts = field.split(tagPairDelimiter);
+ tagQueries.computeIfAbsent(queryParts[0], k -> new ArrayList<>()).add(queryParts[1]);
+ }
+ }
+ if (timestamp == null) {
+ return Status.BAD_REQUEST;
+ }
+
+ return read(table, timestamp, tagQueries);
+ }
+
+ /**
+ * Read a record from the database. Each value from the result will be stored in a HashMap
+ *
+ * @param metric The name of the metric
+ * @param timestamp The timestamp of the record to read.
+ * @param tags actual tags that were want to receive (can be empty)
+ * @return Zero on success, a non-zero error code on error or "not found".
+ */
+ protected abstract Status read(String metric, long timestamp, Map> tags);
+
+ /**
+ * @inheritDoc
+ * @implNote this method parses the information passed to it and subsequently passes it to the modified
+ * interface at {@link #scan(String, long, long, Map, AggregationOperation, int, TimeUnit)}
+ */
+ @Override
+ public final Status scan(String table, String startkey, int recordcount, Set fields,
+ Vector> result) {
+ Map> tagQueries = new HashMap<>();
+ TimeseriesDB.AggregationOperation aggregationOperation = TimeseriesDB.AggregationOperation.NONE;
+ Set groupByFields = new HashSet<>();
+
+ boolean rangeSet = false;
+ long start = 0;
+ long end = 0;
+ for (String field : fields) {
+ if (field.startsWith(timestampKey)) {
+ String[] timestampParts = field.split(tagPairDelimiter);
+ if (!timestampParts[1].contains(queryTimeSpanDelimiter)) {
+ // seems like this should be a more elaborate query.
+ // for now we don't support scanning single timestamps
+ // TODO: Support Timestamp range queries
+ return Status.NOT_IMPLEMENTED;
+ }
+ String[] rangeParts = timestampParts[1].split(queryTimeSpanDelimiter);
+ rangeSet = true;
+ start = Long.valueOf(rangeParts[0]);
+ end = Long.valueOf(rangeParts[1]);
+ } else if (field.startsWith(groupByKey)) {
+ String groupBySpecifier = field.split(tagPairDelimiter)[1];
+ aggregationOperation = TimeseriesDB.AggregationOperation.valueOf(groupBySpecifier);
+ } else if (field.startsWith(downsamplingKey)) {
+ String downsamplingSpec = field.split(tagPairDelimiter)[1];
+ // apparently that needs to always hold true:
+ if (!downsamplingSpec.equals(downsamplingFunction.toString() + downsamplingInterval.toString())) {
+ System.err.print("Downsampling specification for Scan did not match configured downsampling");
+ return Status.BAD_REQUEST;
+ }
+ } else {
+ String[] queryParts = field.split(tagPairDelimiter);
+ if (queryParts.length == 1) {
+ // we should probably warn about this being ignored...
+ System.err.println("Grouping by arbitrary series is currently not supported");
+ groupByFields.add(field);
+ } else {
+ tagQueries.computeIfAbsent(queryParts[0], k -> new ArrayList<>()).add(queryParts[1]);
+ }
+ }
+ }
+ if (!rangeSet) {
+ return Status.BAD_REQUEST;
+ }
+ return scan(table, start, end, tagQueries, downsamplingFunction, downsamplingInterval, timestampUnit);
+ }
+
+ /**
+ * Perform a range scan for a set of records in the database. Each value from the result will be stored in a
+ * HashMap.
+ *
+ * @param metric The name of the metric
+ * @param startTs The timestamp of the first record to read.
+ * @param endTs The timestamp of the last record to read.
+ * @param tags actual tags that were want to receive (can be empty).
+ * @param aggreg The aggregation operation to perform.
+ * @param timeValue value for timeUnit for aggregation
+ * @param timeUnit timeUnit for aggregation
+ * @return A {@link Status} detailing the outcome of the scan operation.
+ */
+ protected abstract Status scan(String metric, long startTs, long endTs, Map> tags,
+ AggregationOperation aggreg, int timeValue, TimeUnit timeUnit);
+
+ @Override
+ public Status update(String table, String key, Map values) {
+ return Status.NOT_IMPLEMENTED;
+ // not supportable for general TSDBs
+ // can be explicitly overwritten in inheriting classes
+ }
+
+ @Override
+ public final Status insert(String table, String key, List fields) {
+ NumericByteIterator tsContainer = null;
+ NumericByteIterator valueContainer = null;
+ Map values = new HashMap();
+ for(DatabaseField f : fields) {
+ String fieldname = f.getFieldname();
+ if(timestampKey.equals(fieldname)) {
+ tsContainer = (NumericByteIterator) f.getContent().asIterator();
+ continue;
+ }
+ if(valueKey.equals(f.getFieldname())) {
+ valueContainer = (NumericByteIterator) f.getContent().asIterator();
+ continue;
+ }
+ values.put(fieldname, f.getContent().asIterator());
+ }
+ if (valueContainer.isFloatingPoint()) {
+ return insert(table, tsContainer.getLong(), valueContainer.getDouble(), values);
+ } else {
+ return insert(table, tsContainer.getLong(), valueContainer.getLong(), values);
+ }
+ }
+
+ /**
+ * Insert a record into the database. Any tags/tagvalue pairs in the specified tagmap and the given value will be
+ * written into the record with the specified timestamp.
+ *
+ * @param metric The name of the metric
+ * @param timestamp The timestamp of the record to insert.
+ * @param value The actual value to insert.
+ * @param tags A Map of tag/tagvalue pairs to insert as tags
+ * @return A {@link Status} detailing the outcome of the insert
+ */
+ protected abstract Status insert(String metric, long timestamp, long value, Map tags);
+
+ /**
+ * Insert a record in the database. Any tags/tagvalue pairs in the specified tagmap and the given value will be
+ * written into the record with the specified timestamp.
+ *
+ * @param metric The name of the metric
+ * @param timestamp The timestamp of the record to insert.
+ * @param value actual value to insert
+ * @param tags A HashMap of tag/tagvalue pairs to insert as tags
+ * @return A {@link Status} detailing the outcome of the insert
+ */
+ protected abstract Status insert(String metric, long timestamp, double value, Map tags);
+
+ /**
+ * NOTE: This operation is usually not supported for Time-Series databases.
+ * Deletion of data is often instead regulated through automatic cleanup and "retention policies" or similar.
+ *
+ * @return Status.NOT_IMPLEMENTED or a {@link Status} specifying the outcome of deletion
+ * in case the operation is supported.
+ */
+ public Status delete(String table, String key) {
+ return Status.NOT_IMPLEMENTED;
+ }
+
+ /**
+ * Examines the given {@link Properties} and returns an array containing the Tag Keys
+ * (basically matching column names for traditional Relational DBs) that are detailed in the workload specification.
+ * See {@link TimeSeriesWorkload} for how these are generated.
+ *
+ * This method is intended to be called during the initialization phase to create a table schema
+ * for DBMS that require such a schema before values can be inserted (or queried)
+ *
+ * @param properties The properties detailing the workload configuration.
+ * @return An array of strings specifying all allowed TagKeys (or column names)
+ * except for the "value" and the "timestamp" column name.
+ * @implSpec WARNING this method must exactly match how tagKeys are generated by the {@link TimeSeriesWorkload},
+ * otherwise databases requiring this information will most likely break!
+ */
+ protected static String[] getPossibleTagKeys(Properties properties) {
+ final int tagCount = Integer.parseInt(properties.getProperty(TimeSeriesWorkload.TAG_COUNT_PROPERTY,
+ TimeSeriesWorkload.TAG_COUNT_PROPERTY_DEFAULT));
+ final int tagKeylength = Integer.parseInt(properties.getProperty(TimeSeriesWorkload.TAG_KEY_LENGTH_PROPERTY,
+ TimeSeriesWorkload.TAG_KEY_LENGTH_PROPERTY_DEFAULT));
+
+ Generator tagKeyGenerator = new IncrementingPrintableStringGenerator(tagKeylength);
+ String[] tagNames = new String[tagCount];
+ for (int i = 0; i < tagCount; i++) {
+ tagNames[i] = tagKeyGenerator.nextValue();
+ }
+ return tagNames;
+ }
+
+
+ /**
+ * An enum containing the possible aggregation operations.
+ * Not all of these operations are required to be supported by implementing classes.
+ *
+ * Aggregations are applied when using the SCAN operation on a range of timestamps.
+ * That way the result set is reduced from multiple records into
+ * a single one or one record for each group specified through GROUP BY clauses.
+ */
+ public enum AggregationOperation {
+ /**
+ * No aggregation whatsoever. Return the results as a full table
+ */
+ NONE,
+ /**
+ * Sum the values of the matching records when calculating the value.
+ * GroupBy criteria apply where relevant for sub-summing.
+ */
+ SUM,
+ /**
+ * Calculate the arithmetic mean over the value across matching records when calculating the value.
+ * GroupBy criteria apply where relevant for group-targeted averages
+ */
+ AVERAGE,
+ /**
+ * Count the number of matching records and return that as value.
+ * GroupBy criteria apply where relevant.
+ */
+ COUNT,
+ /**
+ * Return only the maximum of the matching record values.
+ * GroupBy criteria apply and result in group-based maxima.
+ */
+ MAX,
+ /**
+ * Return only the minimum of the matching record values.
+ * GroupBy criteria apply and result in group-based minima.
+ */
+ MIN;
+ }
+}
diff --git a/core/src/main/java/site/ycsb/UnknownDBException.java b/core/src/main/java/site/ycsb/UnknownDBException.java
new file mode 100644
index 0000000..1660ea7
--- /dev/null
+++ b/core/src/main/java/site/ycsb/UnknownDBException.java
@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+/**
+ * Could not create the specified DB.
+ */
+public class UnknownDBException extends Exception {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 459099842269616836L;
+
+ public UnknownDBException(String message) {
+ super(message);
+ }
+
+ public UnknownDBException() {
+ super();
+ }
+
+ public UnknownDBException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public UnknownDBException(Throwable cause) {
+ super(cause);
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/Utils.java b/core/src/main/java/site/ycsb/Utils.java
new file mode 100644
index 0000000..6bfb4fa
--- /dev/null
+++ b/core/src/main/java/site/ycsb/Utils.java
@@ -0,0 +1,232 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc., 2016 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Utility functions.
+ */
+public final class Utils {
+ private Utils() {
+ // not used
+ }
+
+ /**
+ * Hash an integer value.
+ */
+ public static long hash(long val) {
+ return fnvhash64(val);
+ }
+
+ public static final long FNV_OFFSET_BASIS_64 = 0xCBF29CE484222325L;
+ public static final long FNV_PRIME_64 = 1099511628211L;
+
+ /**
+ * 64 bit FNV hash. Produces more "random" hashes than (say) String.hashCode().
+ *
+ * @param val The value to hash.
+ * @return The hash value
+ */
+ public static long fnvhash64(long val) {
+ //from http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash
+ long hashval = FNV_OFFSET_BASIS_64;
+
+ for (int i = 0; i < 8; i++) {
+ long octet = val & 0x00ff;
+ val = val >> 8;
+
+ hashval = hashval ^ octet;
+ hashval = hashval * FNV_PRIME_64;
+ //hashval = hashval ^ octet;
+ }
+ return Math.abs(hashval);
+ }
+
+ /**
+ * Reads a big-endian 8-byte long from an offset in the given array.
+ * @param bytes The array to read from.
+ * @return A long integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ * @throws NullPointerException if the byte array is null.
+ */
+ public static long bytesToLong(final byte[] bytes) {
+ return (bytes[0] & 0xFFL) << 56
+ | (bytes[1] & 0xFFL) << 48
+ | (bytes[2] & 0xFFL) << 40
+ | (bytes[3] & 0xFFL) << 32
+ | (bytes[4] & 0xFFL) << 24
+ | (bytes[5] & 0xFFL) << 16
+ | (bytes[6] & 0xFFL) << 8
+ | (bytes[7] & 0xFFL) << 0;
+ }
+
+ /**
+ * Writes a big-endian 8-byte long at an offset in the given array.
+ * @param val The value to encode.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static byte[] longToBytes(final long val) {
+ final byte[] bytes = new byte[8];
+ bytes[0] = (byte) (val >>> 56);
+ bytes[1] = (byte) (val >>> 48);
+ bytes[2] = (byte) (val >>> 40);
+ bytes[3] = (byte) (val >>> 32);
+ bytes[4] = (byte) (val >>> 24);
+ bytes[5] = (byte) (val >>> 16);
+ bytes[6] = (byte) (val >>> 8);
+ bytes[7] = (byte) (val >>> 0);
+ return bytes;
+ }
+
+ /**
+ * Parses the byte array into a double.
+ * The byte array must be at least 8 bytes long and have been encoded using
+ * {@link #doubleToBytes}. If the array is longer than 8 bytes, only the
+ * first 8 bytes are parsed.
+ * @param bytes The byte array to parse, at least 8 bytes.
+ * @return A double value read from the byte array.
+ * @throws IllegalArgumentException if the byte array is not 8 bytes wide.
+ */
+ public static double bytesToDouble(final byte[] bytes) {
+ if (bytes.length < 8) {
+ throw new IllegalArgumentException("Byte array must be 8 bytes wide.");
+ }
+ return Double.longBitsToDouble(bytesToLong(bytes));
+ }
+
+ /**
+ * Encodes the double value as an 8 byte array.
+ * @param val The double value to encode.
+ * @return A byte array of length 8.
+ */
+ public static byte[] doubleToBytes(final double val) {
+ return longToBytes(Double.doubleToRawLongBits(val));
+ }
+
+ /**
+ * Measure the estimated active thread count in the current thread group.
+ * Since this calls {@link Thread.activeCount} it should be called from the
+ * main thread or one started by the main thread. Threads included in the
+ * count can be in any state.
+ * For a more accurate count we could use {@link Thread.getAllStackTraces().size()}
+ * but that freezes the JVM and incurs a high overhead.
+ * @return An estimated thread count, good for showing the thread count
+ * over time.
+ */
+ public static int getActiveThreadCount() {
+ return Thread.activeCount();
+ }
+
+ /** @return The currently used memory in bytes */
+ public static long getUsedMemoryBytes() {
+ final Runtime runtime = Runtime.getRuntime();
+ return runtime.totalMemory() - runtime.freeMemory();
+ }
+
+ /** @return The currently used memory in megabytes. */
+ public static int getUsedMemoryMegaBytes() {
+ return (int) (getUsedMemoryBytes() / 1024 / 1024);
+ }
+
+ /** @return The current system load average if supported by the JDK.
+ * If it's not supported, the value will be negative. */
+ public static double getSystemLoadAverage() {
+ final OperatingSystemMXBean osBean =
+ ManagementFactory.getOperatingSystemMXBean();
+ return osBean.getSystemLoadAverage();
+ }
+
+ /** @return The total number of garbage collections executed for all
+ * memory pools. */
+ public static long getGCTotalCollectionCount() {
+ final List gcBeans =
+ ManagementFactory.getGarbageCollectorMXBeans();
+ long count = 0;
+ for (final GarbageCollectorMXBean bean : gcBeans) {
+ if (bean.getCollectionCount() < 0) {
+ continue;
+ }
+ count += bean.getCollectionCount();
+ }
+ return count;
+ }
+
+ /** @return The total time, in milliseconds, spent in GC. */
+ public static long getGCTotalTime() {
+ final List gcBeans =
+ ManagementFactory.getGarbageCollectorMXBeans();
+ long time = 0;
+ for (final GarbageCollectorMXBean bean : gcBeans) {
+ if (bean.getCollectionTime() < 0) {
+ continue;
+ }
+ time += bean.getCollectionTime();
+ }
+ return time;
+ }
+
+ /**
+ * Returns a map of garbage collectors and their stats.
+ * The first object in the array is the total count since JVM start and the
+ * second is the total time (ms) since JVM start.
+ * If a garbage collectors does not support the collector MXBean, then it
+ * will not be represented in the map.
+ * @return A non-null map of garbage collectors and their metrics. The map
+ * may be empty.
+ */
+ public static Map getGCStatst() {
+ final List gcBeans =
+ ManagementFactory.getGarbageCollectorMXBeans();
+ final Map map = new HashMap(gcBeans.size());
+ for (final GarbageCollectorMXBean bean : gcBeans) {
+ if (!bean.isValid() || bean.getCollectionCount() < 0 ||
+ bean.getCollectionTime() < 0) {
+ continue;
+ }
+
+ final Long[] measurements = new Long[]{
+ bean.getCollectionCount(),
+ bean.getCollectionTime()
+ };
+ map.put(bean.getName().replace(" ", "_"), measurements);
+ }
+ return map;
+ }
+
+ /**
+ * Simple Fisher-Yates array shuffle to randomize discrete sets.
+ * @param array The array to randomly shuffle.
+ * @return The shuffled array.
+ */
+ public static T [] shuffleArray(final T[] array) {
+ for (int i = array.length -1; i > 0; i--) {
+ final int idx = ThreadLocalRandom.current().nextInt(i + 1);
+ final T temp = array[idx];
+ array[idx] = array[i];
+ array[i] = temp;
+ }
+ return array;
+ }
+}
diff --git a/core/src/main/java/site/ycsb/Workload.java b/core/src/main/java/site/ycsb/Workload.java
new file mode 100644
index 0000000..1d7ae24
--- /dev/null
+++ b/core/src/main/java/site/ycsb/Workload.java
@@ -0,0 +1,122 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.Properties;
+
+
+/**
+ * One experiment scenario. One object of this type will
+ * be instantiated and shared among all client threads. This class
+ * should be constructed using a no-argument constructor, so we can
+ * load it dynamically. Any argument-based initialization should be
+ * done by init().
+ *
+ * If you extend this class, you should support the "insertstart" property. This
+ * allows the Client to proceed from multiple clients on different machines, in case
+ * the client is the bottleneck. For example, if we want to load 1 million records from
+ * 2 machines, the first machine should have insertstart=0 and the second insertstart=500000. Additionally,
+ * the "insertcount" property, which is interpreted by Client, can be used to tell each instance of the
+ * client how many inserts to do. In the example above, both clients should have insertcount=500000.
+ */
+public abstract class Workload {
+ public static final String INSERT_START_PROPERTY = "insertstart";
+ public static final String INSERT_COUNT_PROPERTY = "insertcount";
+
+ public static final String INSERT_START_PROPERTY_DEFAULT = "0";
+
+ private volatile AtomicBoolean stopRequested = new AtomicBoolean(false);
+
+ /** Operations available for a database. */
+ public enum Operation {
+ READ,
+ UPDATE,
+ INSERT,
+ SCAN,
+ DELETE
+ }
+
+ /**
+ * Initialize the scenario. Create any generators and other shared objects here.
+ * Called once, in the main client thread, before any operations are started.
+ */
+ public void init(Properties p) throws WorkloadException {
+ }
+
+ /**
+ * Initialize any state for a particular client thread. Since the scenario object
+ * will be shared among all threads, this is the place to create any state that is specific
+ * to one thread. To be clear, this means the returned object should be created anew on each
+ * call to initThread(); do not return the same object multiple times.
+ * The returned object will be passed to invocations of doInsert() and doTransaction()
+ * for this thread. There should be no side effects from this call; all state should be encapsulated
+ * in the returned object. If you have no state to retain for this thread, return null. (But if you have
+ * no state to retain for this thread, probably you don't need to override initThread().)
+ *
+ * @return false if the workload knows it is done for this thread. Client will terminate the thread.
+ * Return true otherwise. Return true for workloads that rely on operationcount. For workloads that read
+ * traces from a file, return true when there are more to do, false when you are done.
+ */
+ public Object initThread(Properties p, int mythreadid, int threadcount) throws WorkloadException {
+ return null;
+ }
+
+ /**
+ * Cleanup the scenario. Called once, in the main client thread, after all operations have completed.
+ */
+ public void cleanup() throws WorkloadException {
+ }
+
+ /**
+ * Do one insert operation. Because it will be called concurrently from multiple client threads, this
+ * function must be thread safe. However, avoid synchronized, or the threads will block waiting for each
+ * other, and it will be difficult to reach the target throughput. Ideally, this function would have no side
+ * effects other than DB operations and mutations on threadstate. Mutations to threadstate do not need to be
+ * synchronized, since each thread has its own threadstate instance.
+ */
+ public abstract boolean doInsert(DB db, Object threadstate);
+
+ /**
+ * Do one transaction operation. Because it will be called concurrently from multiple client threads, this
+ * function must be thread safe. However, avoid synchronized, or the threads will block waiting for each
+ * other, and it will be difficult to reach the target throughput. Ideally, this function would have no side
+ * effects other than DB operations and mutations on threadstate. Mutations to threadstate do not need to be
+ * synchronized, since each thread has its own threadstate instance.
+ *
+ * @return false if the workload knows it is done for this thread. Client will terminate the thread.
+ * Return true otherwise. Return true for workloads that rely on operationcount. For workloads that read
+ * traces from a file, return true when there are more to do, false when you are done.
+ */
+ public abstract boolean doTransaction(DB db, Object threadstate);
+
+ /**
+ * Allows scheduling a request to stop the workload.
+ */
+ public void requestStop() {
+ stopRequested.set(true);
+ }
+
+ /**
+ * Check the status of the stop request flag.
+ * @return true if stop was requested, false otherwise.
+ */
+ public boolean isStopRequested() {
+ return stopRequested.get();
+ }
+}
diff --git a/core/src/main/java/site/ycsb/WorkloadException.java b/core/src/main/java/site/ycsb/WorkloadException.java
new file mode 100644
index 0000000..61c9986
--- /dev/null
+++ b/core/src/main/java/site/ycsb/WorkloadException.java
@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb;
+
+/**
+ * The workload tried to do something bad.
+ */
+public class WorkloadException extends Exception {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 8844396756042772132L;
+
+ public WorkloadException(String message) {
+ super(message);
+ }
+
+ public WorkloadException() {
+ super();
+ }
+
+ public WorkloadException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public WorkloadException(Throwable cause) {
+ super(cause);
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/generator/ConstantIntegerGenerator.java b/core/src/main/java/site/ycsb/generator/ConstantIntegerGenerator.java
new file mode 100644
index 0000000..34ea04d
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/ConstantIntegerGenerator.java
@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.generator;
+
+/**
+ * A trivial integer generator that always returns the same value.
+ *
+ */
+public class ConstantIntegerGenerator extends NumberGenerator {
+ private final int i;
+
+ /**
+ * @param i The integer that this generator will always return.
+ */
+ public ConstantIntegerGenerator(int i) {
+ this.i = i;
+ }
+
+ @Override
+ public Integer nextValue() {
+ return i;
+ }
+
+ @Override
+ public double mean() {
+ return i;
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/generator/CounterGenerator.java b/core/src/main/java/site/ycsb/generator/CounterGenerator.java
new file mode 100644
index 0000000..01f05fa
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/CounterGenerator.java
@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc., Copyright (c) 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Generates a sequence of integers.
+ * (0, 1, ...)
+ */
+public class CounterGenerator extends NumberGenerator {
+ private final AtomicLong counter;
+
+ /**
+ * Create a counter that starts at countstart.
+ */
+ public CounterGenerator(long countstart) {
+ counter=new AtomicLong(countstart);
+ }
+
+ @Override
+ public Long nextValue() {
+ return counter.getAndIncrement();
+ }
+
+ @Override
+ public Long lastValue() {
+ return counter.get() - 1;
+ }
+
+ @Override
+ public double mean() {
+ throw new UnsupportedOperationException("Can't compute mean of non-stationary distribution!");
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/DiscreteGenerator.java b/core/src/main/java/site/ycsb/generator/DiscreteGenerator.java
new file mode 100644
index 0000000..8d397f3
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/DiscreteGenerator.java
@@ -0,0 +1,90 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Generates a distribution by choosing from a discrete set of values.
+ */
+public class DiscreteGenerator extends Generator {
+ private static class Pair {
+ private double weight;
+ private String value;
+
+ Pair(double weight, String value) {
+ this.weight = weight;
+ this.value = requireNonNull(value);
+ }
+ }
+
+ private final Collection values = new ArrayList<>();
+ private String lastvalue;
+
+ public DiscreteGenerator() {
+ lastvalue = null;
+ }
+
+ /**
+ * Generate the next string in the distribution.
+ */
+ @Override
+ public String nextValue() {
+ double sum = 0;
+
+ for (Pair p : values) {
+ sum += p.weight;
+ }
+
+ double val = ThreadLocalRandom.current().nextDouble();
+
+ for (Pair p : values) {
+ double pw = p.weight / sum;
+ if (val < pw) {
+ return p.value;
+ }
+
+ val -= pw;
+ }
+
+ throw new AssertionError("oops. should not get here.");
+
+ }
+
+ /**
+ * Return the previous string generated by the distribution; e.g., returned from the last nextString() call.
+ * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet
+ * been called, lastString() should return something reasonable.
+ */
+ @Override
+ public String lastValue() {
+ if (lastvalue == null) {
+ lastvalue = nextValue();
+ }
+ return lastvalue;
+ }
+
+ public void addValue(double weight, String value) {
+ values.add(new Pair(weight, value));
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/generator/ExponentialGenerator.java b/core/src/main/java/site/ycsb/generator/ExponentialGenerator.java
new file mode 100644
index 0000000..39ead69
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/ExponentialGenerator.java
@@ -0,0 +1,87 @@
+/**
+ * Copyright (c) 2011-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * A generator of an exponential distribution. It produces a sequence
+ * of time intervals according to an exponential
+ * distribution. Smaller intervals are more frequent than larger
+ * ones, and there is no bound on the length of an interval. When you
+ * construct an instance of this class, you specify a parameter gamma,
+ * which corresponds to the rate at which events occur.
+ * Alternatively, 1/gamma is the average length of an interval.
+ */
+public class ExponentialGenerator extends NumberGenerator {
+ // What percentage of the readings should be within the most recent exponential.frac portion of the dataset?
+ public static final String EXPONENTIAL_PERCENTILE_PROPERTY = "exponential.percentile";
+ public static final String EXPONENTIAL_PERCENTILE_DEFAULT = "95";
+
+ // What fraction of the dataset should be accessed exponential.percentile of the time?
+ public static final String EXPONENTIAL_FRAC_PROPERTY = "exponential.frac";
+ public static final String EXPONENTIAL_FRAC_DEFAULT = "0.8571428571"; // 1/7
+
+ /**
+ * The exponential constant to use.
+ */
+ private double gamma;
+
+ /******************************* Constructors **************************************/
+
+ /**
+ * Create an exponential generator with a mean arrival rate of
+ * gamma. (And half life of 1/gamma).
+ */
+ public ExponentialGenerator(double mean) {
+ gamma = 1.0 / mean;
+ }
+
+ public ExponentialGenerator(double percentile, double range) {
+ gamma = -Math.log(1.0 - percentile / 100.0) / range; //1.0/mean;
+ }
+
+ /****************************************************************************************/
+
+
+ /**
+ * Generate the next item as a long. This distribution will be skewed toward lower values; e.g. 0 will
+ * be the most popular, 1 the next most popular, etc.
+ * @return The next item in the sequence.
+ */
+ @Override
+ public Double nextValue() {
+ return -Math.log(ThreadLocalRandom.current().nextDouble()) / gamma;
+ }
+
+ @Override
+ public double mean() {
+ return 1.0 / gamma;
+ }
+
+ public static void main(String[] args) {
+ ExponentialGenerator e = new ExponentialGenerator(90, 100);
+ int j = 0;
+ for (int i = 0; i < 1000; i++) {
+ if (e.nextValue() < 100) {
+ j++;
+ }
+ }
+ System.out.println("Got " + j + " hits. Expect 900");
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/FileGenerator.java b/core/src/main/java/site/ycsb/generator/FileGenerator.java
new file mode 100644
index 0000000..9542356
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/FileGenerator.java
@@ -0,0 +1,74 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * A generator, whose sequence is the lines of a file.
+ */
+public class FileGenerator extends Generator {
+ private final String filename;
+ private String current;
+ private BufferedReader reader;
+
+ /**
+ * Create a FileGenerator with the given file.
+ * @param filename The file to read lines from.
+ */
+ public FileGenerator(String filename) {
+ this.filename = filename;
+ reloadFile();
+ }
+
+ /**
+ * Return the next string of the sequence, ie the next line of the file.
+ */
+ @Override
+ public synchronized String nextValue() {
+ try {
+ current = reader.readLine();
+ return current;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Return the previous read line.
+ */
+ @Override
+ public String lastValue() {
+ return current;
+ }
+
+ /**
+ * Reopen the file to reuse values.
+ */
+ public synchronized void reloadFile() {
+ try (Reader r = reader) {
+ System.err.println("Reload " + filename);
+ reader = new BufferedReader(new FileReader(filename));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/Generator.java b/core/src/main/java/site/ycsb/generator/Generator.java
new file mode 100644
index 0000000..9b880f5
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/Generator.java
@@ -0,0 +1,47 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+/**
+ * An expression that generates a sequence of values, following some distribution (Uniform, Zipfian, Sequential, etc.).
+ */
+public abstract class Generator {
+ /**
+ * Generate the next value in the distribution.
+ */
+ public abstract V nextValue();
+
+ /**
+ * Return the previous value generated by the distribution; e.g., returned from the last {@link Generator#nextValue()}
+ * call.
+ * Calling {@link #lastValue()} should not advance the distribution or have any side effects. If {@link #nextValue()}
+ * has not yet been called, {@link #lastValue()} should return something reasonable.
+ */
+ public abstract V lastValue();
+
+ public final String nextString() {
+ V ret = nextValue();
+ return ret == null ? null : ret.toString();
+ }
+
+ public final String lastString() {
+ V ret = lastValue();
+ return ret == null ? null : ret.toString();
+ }
+}
+
diff --git a/core/src/main/java/site/ycsb/generator/HistogramGenerator.java b/core/src/main/java/site/ycsb/generator/HistogramGenerator.java
new file mode 100644
index 0000000..d63631f
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/HistogramGenerator.java
@@ -0,0 +1,108 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.generator;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Generate integers according to a histogram distribution. The histogram
+ * buckets are of width one, but the values are multiplied by a block size.
+ * Therefore, instead of drawing sizes uniformly at random within each
+ * bucket, we always draw the largest value in the current bucket, so the value
+ * drawn is always a multiple of blockSize.
+ *
+ * The minimum value this distribution returns is blockSize (not zero).
+ *
+ */
+public class HistogramGenerator extends NumberGenerator {
+
+ private final long blockSize;
+ private final long[] buckets;
+ private long area;
+ private long weightedArea = 0;
+ private double meanSize = 0;
+
+ public HistogramGenerator(String histogramfile) throws IOException {
+ try (BufferedReader in = new BufferedReader(new FileReader(histogramfile))) {
+ String str;
+ String[] line;
+
+ ArrayList a = new ArrayList<>();
+
+ str = in.readLine();
+ if (str == null) {
+ throw new IOException("Empty input file!\n");
+ }
+ line = str.split("\t");
+ if (line[0].compareTo("BlockSize") != 0) {
+ throw new IOException("First line of histogram is not the BlockSize!\n");
+ }
+ blockSize = Integer.parseInt(line[1]);
+
+ while ((str = in.readLine()) != null) {
+ // [0] is the bucket, [1] is the value
+ line = str.split("\t");
+
+ a.add(Integer.parseInt(line[0]), Integer.parseInt(line[1]));
+ }
+ buckets = new long[a.size()];
+ for (int i = 0; i < a.size(); i++) {
+ buckets[i] = a.get(i);
+ }
+ }
+ init();
+ }
+
+ public HistogramGenerator(long[] buckets, int blockSize) {
+ this.blockSize = blockSize;
+ this.buckets = buckets;
+ init();
+ }
+
+ private void init() {
+ for (int i = 0; i < buckets.length; i++) {
+ area += buckets[i];
+ weightedArea += i * buckets[i];
+ }
+ // calculate average file size
+ meanSize = ((double) blockSize) * ((double) weightedArea) / (area);
+ }
+
+ @Override
+ public Long nextValue() {
+ int number = ThreadLocalRandom.current().nextInt((int) area);
+ int i;
+
+ for (i = 0; i < (buckets.length - 1); i++) {
+ number -= buckets[i];
+ if (number <= 0) {
+ return (i + 1) * blockSize;
+ }
+ }
+
+ return i * blockSize;
+ }
+
+ @Override
+ public double mean() {
+ return meanSize;
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/HotspotIntegerGenerator.java b/core/src/main/java/site/ycsb/generator/HotspotIntegerGenerator.java
new file mode 100644
index 0000000..d4b9828
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/HotspotIntegerGenerator.java
@@ -0,0 +1,122 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. Copyright (c) 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.generator;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Generate integers resembling a hotspot distribution where x% of operations
+ * access y% of data items. The parameters specify the bounds for the numbers,
+ * the percentage of the of the interval which comprises the hot set and
+ * the percentage of operations that access the hot set. Numbers of the hot set are
+ * always smaller than any number in the cold set. Elements from the hot set and
+ * the cold set are chose using a uniform distribution.
+ *
+ */
+public class HotspotIntegerGenerator extends NumberGenerator {
+
+ private final long lowerBound;
+ private final long upperBound;
+ private final long hotInterval;
+ private final long coldInterval;
+ private final double hotsetFraction;
+ private final double hotOpnFraction;
+
+ /**
+ * Create a generator for Hotspot distributions.
+ *
+ * @param lowerBound lower bound of the distribution.
+ * @param upperBound upper bound of the distribution.
+ * @param hotsetFraction percentage of data item
+ * @param hotOpnFraction percentage of operations accessing the hot set.
+ */
+ public HotspotIntegerGenerator(long lowerBound, long upperBound,
+ double hotsetFraction, double hotOpnFraction) {
+ if (hotsetFraction < 0.0 || hotsetFraction > 1.0) {
+ System.err.println("Hotset fraction out of range. Setting to 0.0");
+ hotsetFraction = 0.0;
+ }
+ if (hotOpnFraction < 0.0 || hotOpnFraction > 1.0) {
+ System.err.println("Hot operation fraction out of range. Setting to 0.0");
+ hotOpnFraction = 0.0;
+ }
+ if (lowerBound > upperBound) {
+ System.err.println("Upper bound of Hotspot generator smaller than the lower bound. " +
+ "Swapping the values.");
+ long temp = lowerBound;
+ lowerBound = upperBound;
+ upperBound = temp;
+ }
+ this.lowerBound = lowerBound;
+ this.upperBound = upperBound;
+ this.hotsetFraction = hotsetFraction;
+ long interval = upperBound - lowerBound + 1;
+ this.hotInterval = (int) (interval * hotsetFraction);
+ this.coldInterval = interval - hotInterval;
+ this.hotOpnFraction = hotOpnFraction;
+ }
+
+ @Override
+ public Long nextValue() {
+ long value = 0;
+ Random random = ThreadLocalRandom.current();
+ if (random.nextDouble() < hotOpnFraction) {
+ // Choose a value from the hot set.
+ value = lowerBound + Math.abs(random.nextLong()) % hotInterval;
+ } else {
+ // Choose a value from the cold set.
+ value = lowerBound + hotInterval + Math.abs(random.nextLong()) % coldInterval;
+ }
+ setLastValue(value);
+ return value;
+ }
+
+ /**
+ * @return the lowerBound
+ */
+ public long getLowerBound() {
+ return lowerBound;
+ }
+
+ /**
+ * @return the upperBound
+ */
+ public long getUpperBound() {
+ return upperBound;
+ }
+
+ /**
+ * @return the hotsetFraction
+ */
+ public double getHotsetFraction() {
+ return hotsetFraction;
+ }
+
+ /**
+ * @return the hotOpnFraction
+ */
+ public double getHotOpnFraction() {
+ return hotOpnFraction;
+ }
+
+ @Override
+ public double mean() {
+ return hotOpnFraction * (lowerBound + hotInterval / 2.0)
+ + (1 - hotOpnFraction) * (lowerBound + hotInterval + coldInterval / 2.0);
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/IncrementingPrintableStringGenerator.java b/core/src/main/java/site/ycsb/generator/IncrementingPrintableStringGenerator.java
new file mode 100644
index 0000000..3c178d8
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/IncrementingPrintableStringGenerator.java
@@ -0,0 +1,389 @@
+/**
+ * Copyright (c) 2016-2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.generator;
+
+import java.util.*;
+
+/**
+ * A generator that produces strings of {@link #length} using a set of code points
+ * from {@link #characterSet}. Each time {@link #nextValue()} is executed, the string
+ * is incremented by one character. Eventually the string may rollover to the beginning
+ * and the user may choose to have the generator throw a NoSuchElementException at that
+ * point or continue incrementing. (By default the generator will continue incrementing).
+ *
+ * For example, if we set a length of 2 characters and the character set includes
+ * [A, B] then the generator output will be:
+ *
+ *
AA
+ *
AB
+ *
BA
+ *
BB
+ *
AA <-- rolled over
+ *
+ *
+ * This class includes some default character sets to choose from including ASCII
+ * and plane 0 UTF.
+ */
+public class IncrementingPrintableStringGenerator extends Generator {
+
+ /** Default string length for the generator. */
+ public static final int DEFAULTSTRINGLENGTH = 8;
+
+ /**
+ * Set of all character types that include every symbol other than non-printable
+ * control characters.
+ */
+ public static final Set CHAR_TYPES_ALL_BUT_CONTROL;
+
+ static {
+ CHAR_TYPES_ALL_BUT_CONTROL = new HashSet(24);
+ // numbers
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.DECIMAL_DIGIT_NUMBER);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.LETTER_NUMBER);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.OTHER_NUMBER);
+
+ // letters
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.UPPERCASE_LETTER);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.LOWERCASE_LETTER);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.TITLECASE_LETTER);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.OTHER_LETTER);
+
+ // marks
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.COMBINING_SPACING_MARK);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.NON_SPACING_MARK);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.ENCLOSING_MARK);
+
+ // punctuation
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.CONNECTOR_PUNCTUATION);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.DASH_PUNCTUATION);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.START_PUNCTUATION);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.END_PUNCTUATION);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.INITIAL_QUOTE_PUNCTUATION);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.FINAL_QUOTE_PUNCTUATION);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.OTHER_PUNCTUATION);
+
+ // symbols
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.MATH_SYMBOL);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.CURRENCY_SYMBOL);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.MODIFIER_SYMBOL);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.OTHER_SYMBOL);
+
+ // separators
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.SPACE_SEPARATOR);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.LINE_SEPARATOR);
+ CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.PARAGRAPH_SEPARATOR);
+ }
+
+ /**
+ * Set of character types including only decimals, upper and lower case letters.
+ */
+ public static final Set CHAR_TYPES_BASIC_ALPHA;
+
+ static {
+ CHAR_TYPES_BASIC_ALPHA = new HashSet(2);
+ CHAR_TYPES_BASIC_ALPHA.add((int) Character.UPPERCASE_LETTER);
+ CHAR_TYPES_BASIC_ALPHA.add((int) Character.LOWERCASE_LETTER);
+ }
+
+ /**
+ * Set of character types including only decimals, upper and lower case letters.
+ */
+ public static final Set CHAR_TYPES_BASIC_ALPHANUMERICS;
+
+ static {
+ CHAR_TYPES_BASIC_ALPHANUMERICS = new HashSet(3);
+ CHAR_TYPES_BASIC_ALPHANUMERICS.add((int) Character.DECIMAL_DIGIT_NUMBER);
+ CHAR_TYPES_BASIC_ALPHANUMERICS.add((int) Character.UPPERCASE_LETTER);
+ CHAR_TYPES_BASIC_ALPHANUMERICS.add((int) Character.LOWERCASE_LETTER);
+ }
+
+ /**
+ * Set of character types including only decimals, letter numbers,
+ * other numbers, upper, lower, title case as well as letter modifiers
+ * and other letters.
+ */
+ public static final Set CHAR_TYPE_EXTENDED_ALPHANUMERICS;
+
+ static {
+ CHAR_TYPE_EXTENDED_ALPHANUMERICS = new HashSet(8);
+ CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.DECIMAL_DIGIT_NUMBER);
+ CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.LETTER_NUMBER);
+ CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.OTHER_NUMBER);
+ CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.UPPERCASE_LETTER);
+ CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.LOWERCASE_LETTER);
+ CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.TITLECASE_LETTER);
+ CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.MODIFIER_LETTER);
+ CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.OTHER_LETTER);
+ }
+
+ /** The character set to iterate over. */
+ private final int[] characterSet;
+
+ /** An array indices matching a position in the output string. */
+ private int[] indices;
+
+ /** The length of the output string in characters. */
+ private final int length;
+
+ /** The last value returned by the generator. Should be null if {@link #nextValue()}
+ * has not been called.*/
+ private String lastValue;
+
+ /** Whether or not to throw an exception when the string rolls over. */
+ private boolean throwExceptionOnRollover;
+
+ /** Whether or not the generator has rolled over. */
+ private boolean hasRolledOver;
+
+ /**
+ * Generates strings of 8 characters using only the upper and lower case alphabetical
+ * characters from the ASCII set.
+ */
+ public IncrementingPrintableStringGenerator() {
+ this(DEFAULTSTRINGLENGTH, printableBasicAlphaASCIISet());
+ }
+
+ /**
+ * Generates strings of {@link #length} characters using only the upper and lower
+ * case alphabetical characters from the ASCII set.
+ * @param length The length of string to return from the generator.
+ * @throws IllegalArgumentException if the length is less than one.
+ */
+ public IncrementingPrintableStringGenerator(final int length) {
+ this(length, printableBasicAlphaASCIISet());
+ }
+
+ /**
+ * Generates strings of {@link #length} characters using the code points in
+ * {@link #characterSet}.
+ * @param length The length of string to return from the generator.
+ * @param characterSet A set of code points to choose from. Code points in the
+ * set can be in any order, not necessarily lexical.
+ * @throws IllegalArgumentException if the length is less than one or the character
+ * set has fewer than one code points.
+ */
+ public IncrementingPrintableStringGenerator(final int length, final int[] characterSet) {
+ if (length < 1) {
+ throw new IllegalArgumentException("Length must be greater than or equal to 1");
+ }
+ if (characterSet == null || characterSet.length < 1) {
+ throw new IllegalArgumentException("Character set must have at least one character");
+ }
+ this.length = length;
+ this.characterSet = characterSet;
+ indices = new int[length];
+ }
+
+ @Override
+ public String nextValue() {
+ if (hasRolledOver && throwExceptionOnRollover) {
+ throw new NoSuchElementException("The generator has rolled over to the beginning");
+ }
+
+ final StringBuilder buffer = new StringBuilder(length);
+ for (int i = 0; i < length; i++) {
+ buffer.append(Character.toChars(characterSet[indices[i]]));
+ }
+
+ // increment the indices;
+ for (int i = length - 1; i >= 0; --i) {
+ if (indices[i] >= characterSet.length - 1) {
+ indices[i] = 0;
+ if (i == 0 || characterSet.length == 1 && lastValue != null) {
+ hasRolledOver = true;
+ }
+ } else {
+ ++indices[i];
+ break;
+ }
+ }
+
+ lastValue = buffer.toString();
+ return lastValue;
+ }
+
+ @Override
+ public String lastValue() {
+ return lastValue;
+ }
+
+ /** @param exceptionOnRollover Whether or not to throw an exception on rollover. */
+ public void setThrowExceptionOnRollover(final boolean exceptionOnRollover) {
+ this.throwExceptionOnRollover = exceptionOnRollover;
+ }
+
+ /** @return Whether or not to throw an exception on rollover. */
+ public boolean getThrowExceptionOnRollover() {
+ return throwExceptionOnRollover;
+ }
+
+ /**
+ * Returns an array of printable code points with only the upper and lower
+ * case alphabetical characters from the basic ASCII set.
+ * @return An array of code points
+ */
+ public static int[] printableBasicAlphaASCIISet() {
+ final List validCharacters =
+ generatePrintableCharacterSet(0, 127, null, false, CHAR_TYPES_BASIC_ALPHA);
+ final int[] characterSet = new int[validCharacters.size()];
+ for (int i = 0; i < validCharacters.size(); i++) {
+ characterSet[i] = validCharacters.get(i);
+ }
+ return characterSet;
+ }
+
+ /**
+ * Returns an array of printable code points with the upper and lower case
+ * alphabetical characters as well as the numeric values from the basic
+ * ASCII set.
+ * @return An array of code points
+ */
+ public static int[] printableBasicAlphaNumericASCIISet() {
+ final List validCharacters =
+ generatePrintableCharacterSet(0, 127, null, false, CHAR_TYPES_BASIC_ALPHANUMERICS);
+ final int[] characterSet = new int[validCharacters.size()];
+ for (int i = 0; i < validCharacters.size(); i++) {
+ characterSet[i] = validCharacters.get(i);
+ }
+ return characterSet;
+ }
+
+ /**
+ * Returns an array of printable code points with the entire basic ASCII table,
+ * including spaces. Excludes new lines.
+ * @return An array of code points
+ */
+ public static int[] fullPrintableBasicASCIISet() {
+ final List validCharacters =
+ generatePrintableCharacterSet(32, 127, null, false, null);
+ final int[] characterSet = new int[validCharacters.size()];
+ for (int i = 0; i < validCharacters.size(); i++) {
+ characterSet[i] = validCharacters.get(i);
+ }
+ return characterSet;
+ }
+
+ /**
+ * Returns an array of printable code points with the entire basic ASCII table,
+ * including spaces and new lines.
+ * @return An array of code points
+ */
+ public static int[] fullPrintableBasicASCIISetWithNewlines() {
+ final List validCharacters = new ArrayList();
+ validCharacters.add(10); // newline
+ validCharacters.addAll(generatePrintableCharacterSet(32, 127, null, false, null));
+ final int[] characterSet = new int[validCharacters.size()];
+ for (int i = 0; i < validCharacters.size(); i++) {
+ characterSet[i] = validCharacters.get(i);
+ }
+ return characterSet;
+ }
+
+ /**
+ * Returns an array of printable code points the first plane of Unicode characters
+ * including only the alpha-numeric values.
+ * @return An array of code points
+ */
+ public static int[] printableAlphaNumericPlaneZeroSet() {
+ final List validCharacters =
+ generatePrintableCharacterSet(0, 65535, null, false, CHAR_TYPES_BASIC_ALPHANUMERICS);
+ final int[] characterSet = new int[validCharacters.size()];
+ for (int i = 0; i < validCharacters.size(); i++) {
+ characterSet[i] = validCharacters.get(i);
+ }
+ return characterSet;
+ }
+
+ /**
+ * Returns an array of printable code points the first plane of Unicode characters
+ * including all printable characters.
+ * @return An array of code points
+ */
+ public static int[] fullPrintablePlaneZeroSet() {
+ final List validCharacters =
+ generatePrintableCharacterSet(0, 65535, null, false, CHAR_TYPES_ALL_BUT_CONTROL);
+ final int[] characterSet = new int[validCharacters.size()];
+ for (int i = 0; i < validCharacters.size(); i++) {
+ characterSet[i] = validCharacters.get(i);
+ }
+ return characterSet;
+ }
+
+ /**
+ * Generates a list of code points based on a range and filters.
+ * These can be used for generating strings with various ASCII and/or
+ * Unicode printable character sets for use with DBs that may have
+ * character limitations.
+ *
+ * Note that control, surrogate, format, private use and unassigned
+ * code points are skipped.
+ * @param startCodePoint The starting code point, inclusive.
+ * @param lastCodePoint The final code point, inclusive.
+ * @param characterTypesFilter An optional set of allowable character
+ * types. See {@link Character} for types.
+ * @param isFilterAllowableList Determines whether the {@code allowableTypes}
+ * set is inclusive or exclusive. When true, only those code points that
+ * appear in the list will be included in the resulting set. Otherwise
+ * matching code points are excluded.
+ * @param allowableTypes An optional list of code points for inclusion or
+ * exclusion.
+ * @return A list of code points matching the given range and filters. The
+ * list may be empty but is guaranteed not to be null.
+ */
+ public static List generatePrintableCharacterSet(
+ final int startCodePoint,
+ final int lastCodePoint,
+ final Set characterTypesFilter,
+ final boolean isFilterAllowableList,
+ final Set allowableTypes) {
+
+ // since we don't know the final size of the allowable character list we
+ // start with a list then we'll flatten it to an array.
+ final List validCharacters = new ArrayList(lastCodePoint);
+
+ for (int codePoint = startCodePoint; codePoint <= lastCodePoint; ++codePoint) {
+ if (allowableTypes != null &&
+ !allowableTypes.contains(Character.getType(codePoint))) {
+ continue;
+ } else {
+ // skip control points, formats, surrogates, etc
+ final int type = Character.getType(codePoint);
+ if (type == Character.CONTROL ||
+ type == Character.SURROGATE ||
+ type == Character.FORMAT ||
+ type == Character.PRIVATE_USE ||
+ type == Character.UNASSIGNED) {
+ continue;
+ }
+ }
+
+ if (characterTypesFilter != null) {
+ // if the filter is enabled then we need to make sure the code point
+ // is in the allowable list if it's a whitelist or that the code point
+ // is NOT in the list if it's a blacklist.
+ if ((isFilterAllowableList && !characterTypesFilter.contains(codePoint)) ||
+ (characterTypesFilter.contains(codePoint))) {
+ continue;
+ }
+ }
+
+ validCharacters.add(codePoint);
+ }
+ return validCharacters;
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/generator/NumberGenerator.java b/core/src/main/java/site/ycsb/generator/NumberGenerator.java
new file mode 100644
index 0000000..6b42641
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/NumberGenerator.java
@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+/**
+ * A generator that is capable of generating numeric values.
+ *
+ */
+public abstract class NumberGenerator extends Generator {
+ private Number lastVal;
+
+ /**
+ * Set the last value generated. NumberGenerator subclasses must use this call
+ * to properly set the last value, or the {@link #lastValue()} calls won't work.
+ */
+ protected void setLastValue(Number last) {
+ lastVal = last;
+ }
+
+
+ @Override
+ public Number lastValue() {
+ return lastVal;
+ }
+
+ /**
+ * Return the expected value (mean) of the values this generator will return.
+ */
+ public abstract double mean();
+}
diff --git a/core/src/main/java/site/ycsb/generator/RandomDiscreteTimestampGenerator.java b/core/src/main/java/site/ycsb/generator/RandomDiscreteTimestampGenerator.java
new file mode 100644
index 0000000..da3b44a
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/RandomDiscreteTimestampGenerator.java
@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.generator;
+
+import java.util.concurrent.TimeUnit;
+
+import site.ycsb.Utils;
+
+/**
+ * A generator that picks from a discrete set of offsets from a base Unix Epoch
+ * timestamp that returns timestamps in a random order with the guarantee that
+ * each timestamp is only returned once.
+ *
+ * TODO - It would be best to implement some kind of psuedo non-repeating random
+ * generator for this as it's likely OK that some small percentage of values are
+ * repeated. For now we just generate all of the offsets in an array, shuffle
+ * it and then iterate over the array.
+ *
+ * Note that {@link #MAX_INTERVALS} defines a hard limit on the size of the
+ * offset array so that we don't completely blow out the heap.
+ *
+ * The constructor parameter {@code intervals} determines how many values will be
+ * returned by the generator. For example, if the {@code interval} is 60 and the
+ * {@code timeUnits} are set to {@link TimeUnit#SECONDS} and {@code intervals}
+ * is set to 60, then the consumer can call {@link #nextValue()} 60 times for
+ * timestamps within an hour.
+ */
+public class RandomDiscreteTimestampGenerator extends UnixEpochTimestampGenerator {
+
+ /** A hard limit on the size of the offsets array to a void using too much heap. */
+ public static final int MAX_INTERVALS = 16777216;
+
+ /** The total number of intervals for this generator. */
+ private final int intervals;
+
+ // can't be primitives due to the generic params on the sort function :(
+ /** The array of generated offsets from the base time. */
+ private final Integer[] offsets;
+
+ /** The current index into the offsets array. */
+ private int offsetIndex;
+
+ /**
+ * Ctor that uses the current system time as current.
+ * @param interval The interval between timestamps.
+ * @param timeUnits The time units of the returned Unix Epoch timestamp (as well
+ * as the units for the interval).
+ * @param intervals The total number of intervals for the generator.
+ * @throws IllegalArgumentException if the intervals is larger than {@link #MAX_INTERVALS}
+ */
+ public RandomDiscreteTimestampGenerator(final long interval, final TimeUnit timeUnits,
+ final int intervals) {
+ super(interval, timeUnits);
+ this.intervals = intervals;
+ offsets = new Integer[intervals];
+ setup();
+ }
+
+ /**
+ * Ctor for supplying a starting timestamp.
+ * The interval between timestamps.
+ * @param timeUnits The time units of the returned Unix Epoch timestamp (as well
+ * as the units for the interval).
+ * @param startTimestamp The start timestamp to use.
+ * NOTE that this must match the time units used for the interval.
+ * If the units are in nanoseconds, provide a nanosecond timestamp {@code System.nanoTime()}
+ * or in microseconds, {@code System.nanoTime() / 1000}
+ * or in millis, {@code System.currentTimeMillis()}
+ * @param intervals The total number of intervals for the generator.
+ * @throws IllegalArgumentException if the intervals is larger than {@link #MAX_INTERVALS}
+ */
+ public RandomDiscreteTimestampGenerator(final long interval, final TimeUnit timeUnits,
+ final long startTimestamp, final int intervals) {
+ super(interval, timeUnits, startTimestamp);
+ this.intervals = intervals;
+ offsets = new Integer[intervals];
+ setup();
+ }
+
+ /**
+ * Generates the offsets and shuffles the array.
+ */
+ private void setup() {
+ if (intervals > MAX_INTERVALS) {
+ throw new IllegalArgumentException("Too many intervals for the in-memory "
+ + "array. The limit is " + MAX_INTERVALS + ".");
+ }
+ offsetIndex = 0;
+ for (int i = 0; i < intervals; i++) {
+ offsets[i] = i;
+ }
+ Utils.shuffleArray(offsets);
+ }
+
+ @Override
+ public Long nextValue() {
+ if (offsetIndex >= offsets.length) {
+ throw new IllegalStateException("Reached the end of the random timestamp "
+ + "intervals: " + offsetIndex);
+ }
+ lastTimestamp = currentTimestamp;
+ currentTimestamp = startTimestamp + (offsets[offsetIndex++] * getOffset(1));
+ return currentTimestamp;
+ }
+}
\ No newline at end of file
diff --git a/core/src/main/java/site/ycsb/generator/ScrambledZipfianGenerator.java b/core/src/main/java/site/ycsb/generator/ScrambledZipfianGenerator.java
new file mode 100644
index 0000000..155a6cf
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/ScrambledZipfianGenerator.java
@@ -0,0 +1,127 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+import site.ycsb.Utils;
+
+/**
+ * A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than
+ * others, according to a zipfian distribution. When you construct an instance of this class, you specify the number
+ * of items in the set to draw from, either by specifying an itemcount (so that the sequence is of items from 0 to
+ * itemcount-1) or by specifying a min and a max (so that the sequence is of items from min to max inclusive). After
+ * you construct the instance, you can change the number of items by calling nextInt(itemcount) or nextLong(itemcount).
+ *
+ * Unlike @ZipfianGenerator, this class scatters the "popular" items across the itemspace. Use this, instead of
+ * @ZipfianGenerator, if you don't want the head of the distribution (the popular items) clustered together.
+ */
+public class ScrambledZipfianGenerator extends NumberGenerator {
+ public static final double ZETAN = 26.46902820178302;
+ public static final double USED_ZIPFIAN_CONSTANT = 0.99;
+ public static final long ITEM_COUNT = 10000000000L;
+
+ private ZipfianGenerator gen;
+ private final long min, max, itemcount;
+
+ /******************************* Constructors **************************************/
+
+ /**
+ * Create a zipfian generator for the specified number of items.
+ *
+ * @param items The number of items in the distribution.
+ */
+ public ScrambledZipfianGenerator(long items) {
+ this(0, items - 1);
+ }
+
+ /**
+ * Create a zipfian generator for items between min and max.
+ *
+ * @param min The smallest integer to generate in the sequence.
+ * @param max The largest integer to generate in the sequence.
+ */
+ public ScrambledZipfianGenerator(long min, long max) {
+ this(min, max, ZipfianGenerator.ZIPFIAN_CONSTANT);
+ }
+
+ /**
+ * Create a zipfian generator for the specified number of items using the specified zipfian constant.
+ *
+ * @param _items The number of items in the distribution.
+ * @param _zipfianconstant The zipfian constant to use.
+ */
+ /*
+// not supported, as the value of zeta depends on the zipfian constant, and we have only precomputed zeta for one
+zipfian constant
+ public ScrambledZipfianGenerator(long _items, double _zipfianconstant)
+ {
+ this(0,_items-1,_zipfianconstant);
+ }
+*/
+
+ /**
+ * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant. If you
+ * use a zipfian constant other than 0.99, this will take a long time to complete because we need to recompute zeta.
+ *
+ * @param min The smallest integer to generate in the sequence.
+ * @param max The largest integer to generate in the sequence.
+ * @param zipfianconstant The zipfian constant to use.
+ */
+ public ScrambledZipfianGenerator(long min, long max, double zipfianconstant) {
+ this.min = min;
+ this.max = max;
+ itemcount = this.max - this.min + 1;
+ if (zipfianconstant == USED_ZIPFIAN_CONSTANT) {
+ gen = new ZipfianGenerator(0, ITEM_COUNT, zipfianconstant, ZETAN);
+ } else {
+ gen = new ZipfianGenerator(0, ITEM_COUNT, zipfianconstant);
+ }
+ }
+
+ /**************************************************************************************************/
+
+ /**
+ * Return the next long in the sequence.
+ */
+ @Override
+ public Long nextValue() {
+ long ret = gen.nextValue();
+ ret = min + Utils.fnvhash64(ret) % itemcount;
+ setLastValue(ret);
+ return ret;
+ }
+
+ public static void main(String[] args) {
+ double newzetan = ZipfianGenerator.zetastatic(ITEM_COUNT, ZipfianGenerator.ZIPFIAN_CONSTANT);
+ System.out.println("zetan: " + newzetan);
+ System.exit(0);
+
+ ScrambledZipfianGenerator gen = new ScrambledZipfianGenerator(10000);
+
+ for (int i = 0; i < 1000000; i++) {
+ System.out.println("" + gen.nextValue());
+ }
+ }
+
+ /**
+ * since the values are scrambled (hopefully uniformly), the mean is simply the middle of the range.
+ */
+ @Override
+ public double mean() {
+ return ((min) + max) / 2.0;
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/SequentialGenerator.java b/core/src/main/java/site/ycsb/generator/SequentialGenerator.java
new file mode 100644
index 0000000..39ed82e
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/SequentialGenerator.java
@@ -0,0 +1,66 @@
+/**
+ * Copyright (c) 2016-2017 YCSB Contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Generates a sequence of integers 0, 1, ...
+ */
+public class SequentialGenerator extends NumberGenerator {
+ private final AtomicLong counter;
+ private long interval;
+ private long countstart;
+
+ /**
+ * Create a counter that starts at countstart.
+ */
+ public SequentialGenerator(long countstart, long countend) {
+ counter = new AtomicLong();
+ setLastValue(counter.get());
+ this.countstart = countstart;
+ interval = countend - countstart + 1;
+ }
+
+ /**
+ * If the generator returns numeric (long) values, return the next value as an long.
+ * Default is to return -1, which is appropriate for generators that do not return numeric values.
+ */
+ public long nextLong() {
+ long ret = countstart + counter.getAndIncrement() % interval;
+ setLastValue(ret);
+ return ret;
+ }
+
+ @Override
+ public Number nextValue() {
+ long ret = countstart + counter.getAndIncrement() % interval;
+ setLastValue(ret);
+ return ret;
+ }
+
+ @Override
+ public Number lastValue() {
+ return counter.get() + 1;
+ }
+
+ @Override
+ public double mean() {
+ throw new UnsupportedOperationException("Can't compute mean of non-stationary distribution!");
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/SkewedLatestGenerator.java b/core/src/main/java/site/ycsb/generator/SkewedLatestGenerator.java
new file mode 100644
index 0000000..ec18efe
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/SkewedLatestGenerator.java
@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+/**
+ * Generate a popularity distribution of items, skewed to favor recent items significantly more than older items.
+ */
+public class SkewedLatestGenerator extends NumberGenerator {
+ private CounterGenerator basis;
+ private final ZipfianGenerator zipfian;
+
+ public SkewedLatestGenerator(CounterGenerator basis) {
+ this.basis = basis;
+ zipfian = new ZipfianGenerator(this.basis.lastValue());
+ nextValue();
+ }
+
+ /**
+ * Generate the next string in the distribution, skewed Zipfian favoring the items most recently returned by
+ * the basis generator.
+ */
+ @Override
+ public Long nextValue() {
+ long max = basis.lastValue();
+ long next = max - zipfian.nextLong(max);
+ setLastValue(next);
+ return next;
+ }
+
+ public static void main(String[] args) {
+ SkewedLatestGenerator gen = new SkewedLatestGenerator(new CounterGenerator(1000));
+ for (int i = 0; i < Integer.parseInt(args[0]); i++) {
+ System.out.println(gen.nextString());
+ }
+ }
+
+ @Override
+ public double mean() {
+ throw new UnsupportedOperationException("Can't compute mean of non-stationary distribution!");
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/UniformGenerator.java b/core/src/main/java/site/ycsb/generator/UniformGenerator.java
new file mode 100644
index 0000000..b8731f6
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/UniformGenerator.java
@@ -0,0 +1,63 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. Copyright (c) 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * An expression that generates a random value in the specified range.
+ */
+public class UniformGenerator extends Generator {
+ private final List values;
+ private String laststring;
+ private final UniformLongGenerator gen;
+
+ /**
+ * Creates a generator that will return strings from the specified set uniformly randomly.
+ */
+ public UniformGenerator(Collection values) {
+ this.values = new ArrayList<>(values);
+ laststring = null;
+ gen = new UniformLongGenerator(0, values.size() - 1);
+ }
+
+ /**
+ * Generate the next string in the distribution.
+ */
+ @Override
+ public String nextValue() {
+ laststring = values.get(gen.nextValue().intValue());
+ return laststring;
+ }
+
+ /**
+ * Return the previous string generated by the distribution; e.g., returned from the last nextString() call.
+ * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet
+ * been called, lastString() should return something reasonable.
+ */
+ @Override
+ public String lastValue() {
+ if (laststring == null) {
+ nextValue();
+ }
+ return laststring;
+ }
+}
+
diff --git a/core/src/main/java/site/ycsb/generator/UniformLongGenerator.java b/core/src/main/java/site/ycsb/generator/UniformLongGenerator.java
new file mode 100644
index 0000000..d369538
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/UniformLongGenerator.java
@@ -0,0 +1,54 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. Copyright (c) 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Generates longs randomly uniform from an interval.
+ */
+public class UniformLongGenerator extends NumberGenerator {
+ private final long lb, ub, interval;
+
+ /**
+ * Creates a generator that will return longs uniformly randomly from the
+ * interval [lb,ub] inclusive (that is, lb and ub are possible values)
+ * (lb and ub are possible values).
+ *
+ * @param lb the lower bound (inclusive) of generated values
+ * @param ub the upper bound (inclusive) of generated values
+ */
+ public UniformLongGenerator(long lb, long ub) {
+ this.lb = lb;
+ this.ub = ub;
+ interval = this.ub - this.lb + 1;
+ }
+
+ @Override
+ public Long nextValue() {
+ long ret = Math.abs(ThreadLocalRandom.current().nextLong()) % interval + lb;
+ setLastValue(ret);
+
+ return ret;
+ }
+
+ @Override
+ public double mean() {
+ return ((lb + (long) ub)) / 2.0;
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/UnixEpochTimestampGenerator.java b/core/src/main/java/site/ycsb/generator/UnixEpochTimestampGenerator.java
new file mode 100644
index 0000000..47a6101
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/UnixEpochTimestampGenerator.java
@@ -0,0 +1,183 @@
+/**
+ * Copyright (c) 2016-2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A generator that produces Unix epoch timestamps in seconds, milli, micro or
+ * nanoseconds and increments the stamp a given interval each time
+ * {@link #nextValue()} is called. The result is emitted as a long in the same
+ * way calls to {@code System.currentTimeMillis()} and
+ * {@code System.nanoTime()} behave.
+ *
+ * By default, the current system time of the host is used as the starting
+ * timestamp. Calling {@link #initalizeTimestamp(long)} can adjust the timestamp
+ * back or forward in time. For example, if a workload will generate an hour of
+ * data at 1 minute intervals, then to set the start timestamp an hour in the past
+ * from the current run, use:
+ *
+ * A constructor is also present for setting an explicit start time.
+ * Negative intervals are supported as well for iterating back in time.
+ *
+ * WARNING: This generator is not thread safe and should not called from multiple
+ * threads.
+ */
+public class UnixEpochTimestampGenerator extends Generator {
+
+ /** The base timestamp used as a starting reference. */
+ protected long startTimestamp;
+
+ /** The current timestamp that will be incremented. */
+ protected long currentTimestamp;
+
+ /** The last used timestamp. Should always be one interval behind current. */
+ protected long lastTimestamp;
+
+ /** The interval to increment by. Multiplied by {@link #timeUnits}. */
+ protected long interval;
+
+ /** The units of time the interval represents. */
+ protected TimeUnit timeUnits;
+
+ /**
+ * Default ctor with the current system time and a 60 second interval.
+ */
+ public UnixEpochTimestampGenerator() {
+ this(60, TimeUnit.SECONDS);
+ }
+
+ /**
+ * Ctor that uses the current system time as current.
+ * @param interval The interval for incrementing the timestamp.
+ * @param timeUnits The units of time the increment represents.
+ */
+ public UnixEpochTimestampGenerator(final long interval, final TimeUnit timeUnits) {
+ this.interval = interval;
+ this.timeUnits = timeUnits;
+ // move the first timestamp by 1 interval so that the first call to nextValue
+ // returns this timestamp
+ initalizeTimestamp(-1);
+ currentTimestamp -= getOffset(1);
+ lastTimestamp = currentTimestamp;
+ }
+
+ /**
+ * Ctor for supplying a starting timestamp.
+ * @param interval The interval for incrementing the timestamp.
+ * @param timeUnits The units of time the increment represents.
+ * @param startTimestamp The start timestamp to use.
+ * NOTE that this must match the time units used for the interval.
+ * If the units are in nanoseconds, provide a nanosecond timestamp {@code System.nanoTime()}
+ * or in microseconds, {@code System.nanoTime() / 1000}
+ * or in millis, {@code System.currentTimeMillis()}
+ * or seconds and any interval above, {@code System.currentTimeMillis() / 1000}
+ */
+ public UnixEpochTimestampGenerator(final long interval, final TimeUnit timeUnits,
+ final long startTimestamp) {
+ this.interval = interval;
+ this.timeUnits = timeUnits;
+ // move the first timestamp by 1 interval so that the first call to nextValue
+ // returns this timestamp
+ currentTimestamp = startTimestamp - getOffset(1);
+ this.startTimestamp = currentTimestamp;
+ lastTimestamp = currentTimestamp - getOffset(1);
+ }
+
+ /**
+ * Sets the starting timestamp to the current system time plus the interval offset.
+ * E.g. to set the time an hour in the past, supply a value of {@code -60}.
+ * @param intervalOffset The interval to increment or decrement by.
+ */
+ public void initalizeTimestamp(final long intervalOffset) {
+ switch (timeUnits) {
+ case NANOSECONDS:
+ currentTimestamp = System.nanoTime() + getOffset(intervalOffset);
+ break;
+ case MICROSECONDS:
+ currentTimestamp = (System.nanoTime() / 1000) + getOffset(intervalOffset);
+ break;
+ case MILLISECONDS:
+ currentTimestamp = System.currentTimeMillis() + getOffset(intervalOffset);
+ break;
+ case SECONDS:
+ currentTimestamp = (System.currentTimeMillis() / 1000) +
+ getOffset(intervalOffset);
+ break;
+ case MINUTES:
+ currentTimestamp = (System.currentTimeMillis() / 1000) +
+ getOffset(intervalOffset);
+ break;
+ case HOURS:
+ currentTimestamp = (System.currentTimeMillis() / 1000) +
+ getOffset(intervalOffset);
+ break;
+ case DAYS:
+ currentTimestamp = (System.currentTimeMillis() / 1000) +
+ getOffset(intervalOffset);
+ break;
+ default:
+ throw new IllegalArgumentException("Unhandled time unit type: " + timeUnits);
+ }
+ startTimestamp = currentTimestamp;
+ }
+
+ @Override
+ public Long nextValue() {
+ lastTimestamp = currentTimestamp;
+ currentTimestamp += getOffset(1);
+ return currentTimestamp;
+ }
+
+ /**
+ * Returns the proper increment offset to use given the interval and timeunits.
+ * @param intervalOffset The amount of offset to multiply by.
+ * @return An offset value to adjust the timestamp by.
+ */
+ public long getOffset(final long intervalOffset) {
+ switch (timeUnits) {
+ case NANOSECONDS:
+ case MICROSECONDS:
+ case MILLISECONDS:
+ case SECONDS:
+ return intervalOffset * interval;
+ case MINUTES:
+ return intervalOffset * interval * (long) 60;
+ case HOURS:
+ return intervalOffset * interval * (long) (60 * 60);
+ case DAYS:
+ return intervalOffset * interval * (long) (60 * 60 * 24);
+ default:
+ throw new IllegalArgumentException("Unhandled time unit type: " + timeUnits);
+ }
+ }
+
+ @Override
+ public Long lastValue() {
+ return lastTimestamp;
+ }
+
+ /** @return The current timestamp as set by the last call to {@link #nextValue()} */
+ public long currentValue() {
+ return currentTimestamp;
+ }
+
+}
\ No newline at end of file
diff --git a/core/src/main/java/site/ycsb/generator/ZipfianGenerator.java b/core/src/main/java/site/ycsb/generator/ZipfianGenerator.java
new file mode 100644
index 0000000..5e45eaa
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/ZipfianGenerator.java
@@ -0,0 +1,288 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.generator;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than
+ * others, according to a zipfian distribution. When you construct an instance of this class, you specify the number
+ * of items in the set to draw from, either by specifying an itemcount (so that the sequence is of items from 0 to
+ * itemcount-1) or by specifying a min and a max (so that the sequence is of items from min to max inclusive). After
+ * you construct the instance, you can change the number of items by calling nextInt(itemcount) or nextLong(itemcount).
+ *
+ * Note that the popular items will be clustered together, e.g. item 0 is the most popular, item 1 the second most
+ * popular, and so on (or min is the most popular, min+1 the next most popular, etc.) If you don't want this clustering,
+ * and instead want the popular items scattered throughout the item space, then use ScrambledZipfianGenerator instead.
+ *
+ * Be aware: initializing this generator may take a long time if there are lots of items to choose from (e.g. over a
+ * minute for 100 million objects). This is because certain mathematical values need to be computed to properly
+ * generate a zipfian skew, and one of those values (zeta) is a sum sequence from 1 to n, where n is the itemcount.
+ * Note that if you increase the number of items in the set, we can compute a new zeta incrementally, so it should be
+ * fast unless you have added millions of items. However, if you decrease the number of items, we recompute zeta from
+ * scratch, so this can take a long time.
+ *
+ * The algorithm used here is from "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994.
+ */
+public class ZipfianGenerator extends NumberGenerator {
+ public static final double ZIPFIAN_CONSTANT = 0.99;
+
+ /**
+ * Number of items.
+ */
+ private final long items;
+
+ /**
+ * Min item to generate.
+ */
+ private final long base;
+
+ /**
+ * The zipfian constant to use.
+ */
+ private final double zipfianconstant;
+
+ /**
+ * Computed parameters for generating the distribution.
+ */
+ private double alpha, zetan, eta, theta, zeta2theta;
+
+ /**
+ * The number of items used to compute zetan the last time.
+ */
+ private long countforzeta;
+
+ /**
+ * Flag to prevent problems. If you increase the number of items the zipfian generator is allowed to choose from,
+ * this code will incrementally compute a new zeta value for the larger itemcount. However, if you decrease the
+ * number of items, the code computes zeta from scratch; this is expensive for large itemsets.
+ * Usually this is not intentional; e.g. one thread thinks the number of items is 1001 and calls "nextLong()" with
+ * that item count; then another thread who thinks the number of items is 1000 calls nextLong() with itemcount=1000
+ * triggering the expensive recomputation. (It is expensive for 100 million items, not really for 1000 items.) Why
+ * did the second thread think there were only 1000 items? maybe it read the item count before the first thread
+ * incremented it. So this flag allows you to say if you really do want that recomputation. If true, then the code
+ * will recompute zeta if the itemcount goes down. If false, the code will assume itemcount only goes up, and never
+ * recompute.
+ */
+ private boolean allowitemcountdecrease = false;
+
+ /******************************* Constructors **************************************/
+
+ /**
+ * Create a zipfian generator for the specified number of items.
+ * @param items The number of items in the distribution.
+ */
+ public ZipfianGenerator(long items) {
+ this(0, items - 1);
+ }
+
+ /**
+ * Create a zipfian generator for items between min and max.
+ * @param min The smallest integer to generate in the sequence.
+ * @param max The largest integer to generate in the sequence.
+ */
+ public ZipfianGenerator(long min, long max) {
+ this(min, max, ZIPFIAN_CONSTANT);
+ }
+
+ /**
+ * Create a zipfian generator for the specified number of items using the specified zipfian constant.
+ *
+ * @param items The number of items in the distribution.
+ * @param zipfianconstant The zipfian constant to use.
+ */
+ public ZipfianGenerator(long items, double zipfianconstant) {
+ this(0, items - 1, zipfianconstant);
+ }
+
+ /**
+ * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant.
+ * @param min The smallest integer to generate in the sequence.
+ * @param max The largest integer to generate in the sequence.
+ * @param zipfianconstant The zipfian constant to use.
+ */
+ public ZipfianGenerator(long min, long max, double zipfianconstant) {
+ this(min, max, zipfianconstant, zetastatic(max - min + 1, zipfianconstant));
+ }
+
+ /**
+ * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant, using
+ * the precomputed value of zeta.
+ *
+ * @param min The smallest integer to generate in the sequence.
+ * @param max The largest integer to generate in the sequence.
+ * @param zipfianconstant The zipfian constant to use.
+ * @param zetan The precomputed zeta constant.
+ */
+ public ZipfianGenerator(long min, long max, double zipfianconstant, double zetan) {
+
+ items = max - min + 1;
+ base = min;
+ this.zipfianconstant = zipfianconstant;
+
+ theta = this.zipfianconstant;
+
+ zeta2theta = zeta(2, theta);
+
+ alpha = 1.0 / (1.0 - theta);
+ this.zetan = zetan;
+ countforzeta = items;
+ eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / this.zetan);
+
+ nextValue();
+ }
+
+ /**************************************************************************/
+
+ /**
+ * Compute the zeta constant needed for the distribution. Do this from scratch for a distribution with n items,
+ * using the zipfian constant thetaVal. Remember the value of n, so if we change the itemcount, we can recompute zeta.
+ *
+ * @param n The number of items to compute zeta over.
+ * @param thetaVal The zipfian constant.
+ */
+ double zeta(long n, double thetaVal) {
+ countforzeta = n;
+ return zetastatic(n, thetaVal);
+ }
+
+ /**
+ * Compute the zeta constant needed for the distribution. Do this from scratch for a distribution with n items,
+ * using the zipfian constant theta. This is a static version of the function which will not remember n.
+ * @param n The number of items to compute zeta over.
+ * @param theta The zipfian constant.
+ */
+ static double zetastatic(long n, double theta) {
+ return zetastatic(0, n, theta, 0);
+ }
+
+ /**
+ * Compute the zeta constant needed for the distribution. Do this incrementally for a distribution that
+ * has n items now but used to have st items. Use the zipfian constant thetaVal. Remember the new value of
+ * n so that if we change the itemcount, we'll know to recompute zeta.
+ *
+ * @param st The number of items used to compute the last initialsum
+ * @param n The number of items to compute zeta over.
+ * @param thetaVal The zipfian constant.
+ * @param initialsum The value of zeta we are computing incrementally from.
+ */
+ double zeta(long st, long n, double thetaVal, double initialsum) {
+ countforzeta = n;
+ return zetastatic(st, n, thetaVal, initialsum);
+ }
+
+ /**
+ * Compute the zeta constant needed for the distribution. Do this incrementally for a distribution that
+ * has n items now but used to have st items. Use the zipfian constant theta. Remember the new value of
+ * n so that if we change the itemcount, we'll know to recompute zeta.
+ * @param st The number of items used to compute the last initialsum
+ * @param n The number of items to compute zeta over.
+ * @param theta The zipfian constant.
+ * @param initialsum The value of zeta we are computing incrementally from.
+ */
+ static double zetastatic(long st, long n, double theta, double initialsum) {
+ double sum = initialsum;
+ for (long i = st; i < n; i++) {
+
+ sum += 1 / (Math.pow(i + 1, theta));
+ }
+
+ //System.out.println("countforzeta="+countforzeta);
+
+ return sum;
+ }
+
+ /****************************************************************************************/
+
+
+ /**
+ * Generate the next item as a long.
+ *
+ * @param itemcount The number of items in the distribution.
+ * @return The next item in the sequence.
+ */
+ long nextLong(long itemcount) {
+ //from "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994
+
+ if (itemcount != countforzeta) {
+
+ //have to recompute zetan and eta, since they depend on itemcount
+ synchronized (this) {
+ if (itemcount > countforzeta) {
+ //System.err.println("WARNING: Incrementally recomputing Zipfian distribtion. (itemcount="+itemcount+"
+ // countforzeta="+countforzeta+")");
+
+ //we have added more items. can compute zetan incrementally, which is cheaper
+ zetan = zeta(countforzeta, itemcount, theta, zetan);
+ eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan);
+ } else if ((itemcount < countforzeta) && (allowitemcountdecrease)) {
+ //have to start over with zetan
+ //note : for large itemsets, this is very slow. so don't do it!
+
+ //TODO: can also have a negative incremental computation, e.g. if you decrease the number of items,
+ // then just subtract the zeta sequence terms for the items that went away. This would be faster than
+ // recomputing from scratch when the number of items decreases
+
+ System.err.println("WARNING: Recomputing Zipfian distribtion. This is slow and should be avoided. " +
+ "(itemcount=" + itemcount + " countforzeta=" + countforzeta + ")");
+
+ zetan = zeta(itemcount, theta);
+ eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan);
+ }
+ }
+ }
+
+ double u = ThreadLocalRandom.current().nextDouble();
+ double uz = u * zetan;
+
+ if (uz < 1.0) {
+ return base;
+ }
+
+ if (uz < 1.0 + Math.pow(0.5, theta)) {
+ return base + 1;
+ }
+
+ long ret = base + (long) ((itemcount) * Math.pow(eta * u - eta + 1, alpha));
+ setLastValue(ret);
+ return ret;
+ }
+
+ /**
+ * Return the next value, skewed by the Zipfian distribution. The 0th item will be the most popular, followed by
+ * the 1st, followed by the 2nd, etc. (Or, if min != 0, the min-th item is the most popular, the min+1th item the
+ * next most popular, etc.) If you want the popular items scattered throughout the item space, use
+ * ScrambledZipfianGenerator instead.
+ */
+ @Override
+ public Long nextValue() {
+ return nextLong(items);
+ }
+
+ public static void main(String[] args) {
+ new ZipfianGenerator(ScrambledZipfianGenerator.ITEM_COUNT);
+ }
+
+ /**
+ * @todo Implement ZipfianGenerator.mean()
+ */
+ @Override
+ public double mean() {
+ throw new UnsupportedOperationException("@todo implement ZipfianGenerator.mean()");
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/acknowledge/AcknowledgeGeneratorFactory.java b/core/src/main/java/site/ycsb/generator/acknowledge/AcknowledgeGeneratorFactory.java
new file mode 100644
index 0000000..dee8245
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/acknowledge/AcknowledgeGeneratorFactory.java
@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) 2023-204 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.generator.acknowledge;
+
+public final class AcknowledgeGeneratorFactory {
+
+ private AcknowledgeGeneratorFactory() {
+ // private constructor
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/acknowledge/AcknowledgeTester.java b/core/src/main/java/site/ycsb/generator/acknowledge/AcknowledgeTester.java
new file mode 100644
index 0000000..85d1298
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/acknowledge/AcknowledgeTester.java
@@ -0,0 +1,31 @@
+package site.ycsb.generator.acknowledge;
+
+public class AcknowledgeTester {
+
+ public static void main(String[] args) {
+ System.out.println("starting");
+ DefaultAcknowledgedCounterGenerator am = new DefaultAcknowledgedCounterGenerator(10500000000L);
+ long l = am.lastValue();
+ System.out.println("getting values");
+ for(long i = 0; i < (1 << 40); i++) {
+ am.nextValue();
+ }
+ System.out.println("acknowledging pt1");
+ for(long i = 0; i < (1 << 10); i++) {
+ am.acknowledge(l + i);
+ }
+ System.out.println("acknowledging pt2");
+ for(long i = (1 << 30); i < (1 << 40); i++) {
+ am.acknowledge(l + i);
+ }
+ System.out.println("acknowledging pt3");
+ for(long i = (1 << 20); i < (1 << 30); i++) {
+ am.acknowledge(l + i);
+ }
+ System.out.println("acknowledging pt4");
+ for(long i = (1 << 10); i < (1 << 20); i++) {
+ am.acknowledge(l + i);
+ }
+ System.out.println("done");
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/acknowledge/AcknowledgedCounterGenerator.java b/core/src/main/java/site/ycsb/generator/acknowledge/AcknowledgedCounterGenerator.java
new file mode 100644
index 0000000..daab31e
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/acknowledge/AcknowledgedCounterGenerator.java
@@ -0,0 +1,26 @@
+/**
+ * Copyright (c) 2023-204 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.generator.acknowledge;
+
+import site.ycsb.generator.CounterGenerator;
+
+public abstract class AcknowledgedCounterGenerator extends CounterGenerator {
+ protected AcknowledgedCounterGenerator(long countstart) {
+ super(countstart);
+ }
+ public abstract void acknowledge(long value);
+}
diff --git a/core/src/main/java/site/ycsb/generator/acknowledge/DefaultAcknowledgedCounterGenerator.java b/core/src/main/java/site/ycsb/generator/acknowledge/DefaultAcknowledgedCounterGenerator.java
new file mode 100644
index 0000000..f44ef15
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/acknowledge/DefaultAcknowledgedCounterGenerator.java
@@ -0,0 +1,124 @@
+/**
+ * Copyright (c) 2015-2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.generator.acknowledge;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * A CounterGenerator that reports generated integers via lastInt()
+ * only after they have been acknowledged.
+ */
+public class DefaultAcknowledgedCounterGenerator extends AcknowledgedCounterGenerator {
+ /** The size of the window of pending id ack's. 2^20 = {@value} */
+ static final int DEFAUL_WINDOW_SIZE = Integer.rotateLeft(1, 17);
+
+ private final ConcurrentHashMap mapping = new ConcurrentHashMap<>();
+ private final ReentrantLock lock;
+ private final boolean[] window;
+ private volatile long limit;
+ private final int windowSize;
+ /** The mask to use to turn an id into a slot in {@link #window}. */
+ private final int windowMask;
+
+ /**
+ * Create a counter that starts at countstart.
+ */
+ public DefaultAcknowledgedCounterGenerator(long countstart, int windowSize) {
+ super(countstart);
+ if(windowSize < 1) throw new IllegalArgumentException("windo must be positive");
+ this.windowSize = windowSize;
+ lock = new ReentrantLock();
+ window = new boolean[this.windowSize];
+ windowMask = this.windowSize - 1;
+ limit = countstart - 1;
+ System.err.println("starting AcknowledgedCounterGenerator with limit " + limit);
+ }
+
+ /**
+ * Create a counter that starts at countstart.
+ */
+ public DefaultAcknowledgedCounterGenerator(long countstart) {
+ this(countstart, DEFAUL_WINDOW_SIZE);
+ }
+
+ /**
+ * In this generator, the highest acknowledged counter value
+ * (as opposed to the highest generated counter value).
+ */
+ @Override
+ public Long lastValue() {
+ return limit;
+ }
+ @Override
+ public Long nextValue() {
+ Long l = super.nextValue();
+ mapping.put(l, Thread.currentThread().getName());
+ if(l - limit > this.windowSize) {
+ System.err.println("given out more elements than fit into window. Where did they go? Limit is " + limit);
+ }
+ return l;
+ }
+
+ /**
+ * Make a generated counter value available via lastInt().
+ */
+ public void acknowledge(long value) {
+ final int currentSlot = (int)(value & this.windowMask);
+ if (window[currentSlot]) {
+ System.err.println(mapping);
+ System.err.println(lock.isLocked() + " --- " + lock.getHoldCount() + " --- " + lock.getQueueLength());
+ Map map = Thread.getAllStackTraces();
+ Exception e = new Exception();
+ for(Thread t : map.keySet()) {
+ System.err.println("thread: " + t.getName());
+ e.setStackTrace(map.get(t));
+ e.printStackTrace(System.err);
+ }
+ throw new RuntimeException("Too many unacknowledged insertion keys. Limit is " + limit);
+ }
+ mapping.remove(value);
+ window[currentSlot] = true;
+
+ if (lock.tryLock()) {
+ // long start = System.currentTimeMillis();
+ // move a contiguous sequence from the window
+ // over to the "limit" variable
+ try {
+ // Only loop through the entire window at most once.
+ long beforeFirstSlot = (limit & this.windowMask);
+ long index;
+ for (index = limit + 1; index != beforeFirstSlot; ++index) {
+ int slot = (int)(index & this.windowMask);
+ if (!window[slot]) {
+ break;
+ }
+ window[slot] = false;
+ }
+ limit = index - 1;
+ } finally {
+ lock.unlock();
+ /* long end = System.currentTimeMillis();
+ if(end - start > 0) {
+ System.out.println("cleanup took: " + (end -start) + " milliseconds ");
+ }
+ */
+ }
+ }
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/acknowledge/StupidAcknowledgedCounterGenerator.java b/core/src/main/java/site/ycsb/generator/acknowledge/StupidAcknowledgedCounterGenerator.java
new file mode 100644
index 0000000..30d8f09
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/acknowledge/StupidAcknowledgedCounterGenerator.java
@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) 2023-2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.generator.acknowledge;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A CounterGenerator that reports naively reports the
+ * highest acknowledged id without considering holes in
+ * the sequence. This is sufficient for bulk loads.
+ */
+public class StupidAcknowledgedCounterGenerator extends AcknowledgedCounterGenerator {
+
+ private final AtomicLong limit;
+
+ /**
+ * Create a counter that starts at countstart.
+ */
+ public StupidAcknowledgedCounterGenerator(long countstart) {
+ super(countstart);
+ limit = new AtomicLong(countstart - 1);
+ }
+
+ /**
+ * In this generator, the highest acknowledged counter value
+ * (as opposed to the highest generated counter value).
+ */
+ @Override
+ public Long lastValue() {
+ return limit.get();
+ }
+
+ /**
+ * Make a generated counter value available via lastInt().
+ */
+ public void acknowledge(long value) {
+ while(true) {
+ long l = limit.get();
+ if(l < value) {
+ boolean b = limit.compareAndSet(l, value);
+ if(!b) continue;
+ }
+ return;
+ }
+ }
+}
diff --git a/core/src/main/java/site/ycsb/generator/package-info.java b/core/src/main/java/site/ycsb/generator/package-info.java
new file mode 100644
index 0000000..853437f
--- /dev/null
+++ b/core/src/main/java/site/ycsb/generator/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+/**
+ * The YCSB generator package.
+ */
+package site.ycsb.generator;
+
diff --git a/core/src/main/java/site/ycsb/measurements/Measurements.java b/core/src/main/java/site/ycsb/measurements/Measurements.java
new file mode 100644
index 0000000..b6e68bb
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/Measurements.java
@@ -0,0 +1,281 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2020 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.measurements;
+
+import site.ycsb.Status;
+import site.ycsb.measurements.exporter.MeasurementsExporter;
+
+import java.io.IOException;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Collects latency measurements, and reports them when requested.
+ */
+public class Measurements {
+ /**
+ * All supported measurement types are defined in this enum.
+ */
+ public enum MeasurementType {
+ HISTOGRAM,
+ HDRHISTOGRAM,
+ HDRHISTOGRAM_AND_HISTOGRAM,
+ HDRHISTOGRAM_AND_RAW,
+ TIMESERIES,
+ RAW
+ }
+
+ public static final String MEASUREMENT_TYPE_PROPERTY = "measurementtype";
+ private static final String MEASUREMENT_TYPE_PROPERTY_DEFAULT = "hdrhistogram";
+
+ public static final String MEASUREMENT_INTERVAL = "measurement.interval";
+ private static final String MEASUREMENT_INTERVAL_DEFAULT = "op";
+
+ public static final String MEASUREMENT_TRACK_JVM_PROPERTY = "measurement.trackjvm";
+ public static final String MEASUREMENT_TRACK_JVM_PROPERTY_DEFAULT = "false";
+
+ private static Measurements singleton = null;
+ private static Properties measurementproperties = null;
+
+ public static void setProperties(Properties props) {
+ measurementproperties = props;
+ }
+
+ /**
+ * Return the singleton Measurements object.
+ */
+ public static synchronized Measurements getMeasurements() {
+ if (singleton == null) {
+ singleton = new Measurements(measurementproperties);
+ }
+ return singleton;
+ }
+
+ private final ConcurrentHashMap opToMesurementMap;
+ private final ConcurrentHashMap opToIntendedMesurementMap;
+ private final MeasurementType measurementType;
+ private final int measurementInterval;
+ private final Properties props;
+
+ /**
+ * Create a new object with the specified properties.
+ */
+ public Measurements(Properties props) {
+ opToMesurementMap = new ConcurrentHashMap<>();
+ opToIntendedMesurementMap = new ConcurrentHashMap<>();
+
+ this.props = props;
+
+ String mTypeString = this.props.getProperty(MEASUREMENT_TYPE_PROPERTY, MEASUREMENT_TYPE_PROPERTY_DEFAULT);
+ switch (mTypeString) {
+ case "histogram":
+ measurementType = MeasurementType.HISTOGRAM;
+ break;
+ case "hdrhistogram":
+ measurementType = MeasurementType.HDRHISTOGRAM;
+ break;
+ case "hdrhistogram+histogram":
+ measurementType = MeasurementType.HDRHISTOGRAM_AND_HISTOGRAM;
+ break;
+ case "hdrhistogram+raw":
+ measurementType = MeasurementType.HDRHISTOGRAM_AND_RAW;
+ break;
+ case "timeseries":
+ measurementType = MeasurementType.TIMESERIES;
+ break;
+ case "raw":
+ measurementType = MeasurementType.RAW;
+ break;
+ default:
+ throw new IllegalArgumentException("unknown " + MEASUREMENT_TYPE_PROPERTY + "=" + mTypeString);
+ }
+
+ String mIntervalString = this.props.getProperty(MEASUREMENT_INTERVAL, MEASUREMENT_INTERVAL_DEFAULT);
+ switch (mIntervalString) {
+ case "op":
+ measurementInterval = 0;
+ break;
+ case "intended":
+ measurementInterval = 1;
+ break;
+ case "both":
+ measurementInterval = 2;
+ break;
+ default:
+ throw new IllegalArgumentException("unknown " + MEASUREMENT_INTERVAL + "=" + mIntervalString);
+ }
+ }
+
+ private OneMeasurement constructOneMeasurement(String name) {
+ switch (measurementType) {
+ case HISTOGRAM:
+ return new OneMeasurementHistogram(name, props);
+ case HDRHISTOGRAM:
+ return new OneMeasurementHdrHistogram(name, props);
+ case HDRHISTOGRAM_AND_HISTOGRAM:
+ return new TwoInOneMeasurement(name,
+ new OneMeasurementHdrHistogram("Hdr" + name, props),
+ new OneMeasurementHistogram("Bucket" + name, props));
+ case HDRHISTOGRAM_AND_RAW:
+ return new TwoInOneMeasurement(name,
+ new OneMeasurementHdrHistogram("Hdr" + name, props),
+ new OneMeasurementRaw("Raw" + name, props));
+ case TIMESERIES:
+ return new OneMeasurementTimeSeries(name, props);
+ case RAW:
+ return new OneMeasurementRaw(name, props);
+ default:
+ throw new AssertionError("Impossible to be here. Dead code reached. Bugs?");
+ }
+ }
+
+ static class StartTimeHolder {
+ protected long time;
+
+ long startTime() {
+ if (time == 0) {
+ return System.nanoTime();
+ } else {
+ return time;
+ }
+ }
+ }
+
+ private final ThreadLocal tlIntendedStartTime = new ThreadLocal() {
+ protected StartTimeHolder initialValue() {
+ return new StartTimeHolder();
+ }
+ };
+
+ public void setIntendedStartTimeNs(long time) {
+ if (measurementInterval == 0) {
+ return;
+ }
+ tlIntendedStartTime.get().time = time;
+ }
+
+ public long getIntendedStartTimeNs() {
+ if (measurementInterval == 0) {
+ return 0L;
+ }
+ return tlIntendedStartTime.get().startTime();
+ }
+
+ /**
+ * Report a single value of a single metric. E.g. for read latency, operation="READ" and latency is the measured
+ * value.
+ */
+ public void measure(String operation, int latency) {
+ if (measurementInterval == 1) {
+ return;
+ }
+ try {
+ OneMeasurement m = getOpMeasurement(operation);
+ m.measure(latency);
+ } catch (java.lang.ArrayIndexOutOfBoundsException e) {
+ // This seems like a terribly hacky way to cover up for a bug in the measurement code
+ System.out.println("ERROR: java.lang.ArrayIndexOutOfBoundsException - ignoring and continuing");
+ e.printStackTrace();
+ e.printStackTrace(System.out);
+ }
+ }
+
+ /**
+ * Report a single value of a single metric. E.g. for read latency, operation="READ" and latency is the measured
+ * value.
+ */
+ public void measureIntended(String operation, int latency) {
+ if (measurementInterval == 0) {
+ return;
+ }
+ try {
+ OneMeasurement m = getOpIntendedMeasurement(operation);
+ m.measure(latency);
+ } catch (java.lang.ArrayIndexOutOfBoundsException e) {
+ // This seems like a terribly hacky way to cover up for a bug in the measurement code
+ System.out.println("ERROR: java.lang.ArrayIndexOutOfBoundsException - ignoring and continuing");
+ e.printStackTrace();
+ e.printStackTrace(System.out);
+ }
+ }
+
+ private OneMeasurement getOpMeasurement(String operation) {
+ OneMeasurement m = opToMesurementMap.get(operation);
+ if (m == null) {
+ m = constructOneMeasurement(operation);
+ OneMeasurement oldM = opToMesurementMap.putIfAbsent(operation, m);
+ if (oldM != null) {
+ m = oldM;
+ }
+ }
+ return m;
+ }
+
+ private OneMeasurement getOpIntendedMeasurement(String operation) {
+ OneMeasurement m = opToIntendedMesurementMap.get(operation);
+ if (m == null) {
+ final String name = measurementInterval == 1 ? operation : "Intended-" + operation;
+ m = constructOneMeasurement(name);
+ OneMeasurement oldM = opToIntendedMesurementMap.putIfAbsent(operation, m);
+ if (oldM != null) {
+ m = oldM;
+ }
+ }
+ return m;
+ }
+
+ /**
+ * Report a return code for a single DB operation.
+ */
+ public void reportStatus(final String operation, final Status status) {
+ OneMeasurement m = measurementInterval == 1 ?
+ getOpIntendedMeasurement(operation) :
+ getOpMeasurement(operation);
+ m.reportStatus(status);
+ }
+
+ /**
+ * Export the current measurements to a suitable format.
+ *
+ * @param exporter Exporter representing the type of format to write to.
+ * @throws IOException Thrown if the export failed.
+ */
+ public void exportMeasurements(MeasurementsExporter exporter) throws IOException {
+ for (OneMeasurement measurement : opToMesurementMap.values()) {
+ measurement.exportMeasurements(exporter);
+ }
+ for (OneMeasurement measurement : opToIntendedMesurementMap.values()) {
+ measurement.exportMeasurements(exporter);
+ }
+ }
+
+ /**
+ * Return a one line summary of the measurements.
+ */
+ public synchronized String getSummary() {
+ String ret = "";
+ for (OneMeasurement m : opToMesurementMap.values()) {
+ ret += m.getSummary() + " ";
+ }
+ for (OneMeasurement m : opToIntendedMesurementMap.values()) {
+ ret += m.getSummary() + " ";
+ }
+ return ret;
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/measurements/OneMeasurement.java b/core/src/main/java/site/ycsb/measurements/OneMeasurement.java
new file mode 100644
index 0000000..1bc7d51
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/OneMeasurement.java
@@ -0,0 +1,83 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ * Copyright (c) 2023-204 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.measurements;
+
+import site.ycsb.Status;
+import site.ycsb.measurements.exporter.MeasurementsExporter;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A single measured metric (such as READ LATENCY).
+ */
+public abstract class OneMeasurement {
+
+ private final String name;
+ private final ConcurrentHashMap returncodes;
+
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * @param name measurement name
+ */
+ public OneMeasurement(String name) {
+ this.name = name;
+ this.returncodes = new ConcurrentHashMap<>();
+ }
+
+ public abstract void measure(int latency);
+
+ public abstract String getSummary();
+
+ /**
+ * No need for synchronization, using CHM to deal with that.
+ */
+ public void reportStatus(Status status) {
+ AtomicLong counter = returncodes.get(status);
+
+ if (counter == null) {
+ counter = new AtomicLong();
+ AtomicLong other = returncodes.putIfAbsent(status, counter);
+ if (other != null) {
+ counter = other;
+ }
+ }
+
+ counter.incrementAndGet();
+ }
+
+ /**
+ * Export the current measurements to a suitable format.
+ *
+ * @param exporter Exporter representing the type of format to write to.
+ * @throws IOException Thrown if the export failed.
+ */
+ public abstract void exportMeasurements(MeasurementsExporter exporter) throws IOException;
+
+ protected final void exportStatusCounts(MeasurementsExporter exporter) throws IOException {
+ for (Map.Entry entry : returncodes.entrySet()) {
+ exporter.write(getName(), "Return=" + entry.getKey().getName(), entry.getValue().get());
+ }
+ }
+}
diff --git a/core/src/main/java/site/ycsb/measurements/OneMeasurementHdrHistogram.java b/core/src/main/java/site/ycsb/measurements/OneMeasurementHdrHistogram.java
new file mode 100644
index 0000000..e8ab866
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/OneMeasurementHdrHistogram.java
@@ -0,0 +1,224 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.measurements;
+
+import site.ycsb.measurements.exporter.MeasurementsExporter;
+import org.HdrHistogram.Histogram;
+import org.HdrHistogram.HistogramIterationValue;
+import org.HdrHistogram.HistogramLogWriter;
+import org.HdrHistogram.Recorder;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Take measurements and maintain a HdrHistogram of a given metric, such as READ LATENCY.
+ *
+ */
+public class OneMeasurementHdrHistogram extends OneMeasurement {
+
+ // we need one log per measurement histogram
+ private final PrintStream log;
+ private final HistogramLogWriter histogramLogWriter;
+
+ private final Recorder histogram;
+ private Histogram totalHistogram;
+
+ /**
+ * The name of the property for deciding what percentile values to output.
+ */
+ public static final String PERCENTILES_PROPERTY = "hdrhistogram.percentiles";
+
+ /**
+ * The default value for the hdrhistogram.percentiles property.
+ */
+ public static final String PERCENTILES_PROPERTY_DEFAULT = "95,99";
+
+ /**
+ * The name of the property for determining if we should print out the buckets.
+ */
+ public static final String VERBOSE_PROPERTY = "measurement.histogram.verbose";
+
+ /**
+ * Whether or not to emit the histogram buckets.
+ */
+ private final boolean verbose;
+
+ private final List percentiles;
+
+ public OneMeasurementHdrHistogram(String name, Properties props) {
+ super(name);
+ percentiles = getPercentileValues(props.getProperty(PERCENTILES_PROPERTY, PERCENTILES_PROPERTY_DEFAULT));
+ verbose = Boolean.valueOf(props.getProperty(VERBOSE_PROPERTY, String.valueOf(false)));
+ boolean shouldLog = Boolean.parseBoolean(props.getProperty("hdrhistogram.fileoutput", "false"));
+ if (!shouldLog) {
+ log = null;
+ histogramLogWriter = null;
+ } else {
+ try {
+ final String hdrOutputFilename = props.getProperty("hdrhistogram.output.path", "") + name + ".hdr";
+ log = new PrintStream(new FileOutputStream(hdrOutputFilename), false);
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException("Failed to open hdr histogram output file", e);
+ }
+ histogramLogWriter = new HistogramLogWriter(log);
+ histogramLogWriter.outputComment("[Logging for: " + name + "]");
+ histogramLogWriter.outputLogFormatVersion();
+ long now = System.currentTimeMillis();
+ histogramLogWriter.outputStartTime(now);
+ histogramLogWriter.setBaseTime(now);
+ histogramLogWriter.outputLegend();
+ }
+ histogram = new Recorder(3);
+ }
+
+ /**
+ * It appears latency is reported in micros.
+ * Using {@link Recorder} to support concurrent updates to histogram.
+ */
+ public void measure(int latencyInMicros) {
+ histogram.recordValue(latencyInMicros);
+ }
+
+ /**
+ * This is called from a main thread, on orderly termination.
+ */
+ @Override
+ public void exportMeasurements(MeasurementsExporter exporter) throws IOException {
+ // accumulate the last interval which was not caught by status thread
+ Histogram intervalHistogram = getIntervalHistogramAndAccumulate();
+ if (histogramLogWriter != null) {
+ histogramLogWriter.outputIntervalHistogram(intervalHistogram);
+ // we can close now
+ log.close();
+ }
+ exporter.write(getName(), "Operations", totalHistogram.getTotalCount());
+ exporter.write(getName(), "AverageLatency(us)", totalHistogram.getMean());
+ exporter.write(getName(), "MinLatency(us)", totalHistogram.getMinValue());
+ exporter.write(getName(), "MaxLatency(us)", totalHistogram.getMaxValue());
+
+ for (Double percentile : percentiles) {
+ exporter.write(getName(), ordinal(percentile) + "PercentileLatency(us)",
+ totalHistogram.getValueAtPercentile(percentile));
+ }
+
+ exportStatusCounts(exporter);
+
+ // also export totalHistogram
+ if (verbose) {
+ for (HistogramIterationValue v : totalHistogram.recordedValues()) {
+ int value;
+ if (v.getValueIteratedTo() > (long)Integer.MAX_VALUE) {
+ value = Integer.MAX_VALUE;
+ } else {
+ value = (int)v.getValueIteratedTo();
+ }
+
+ exporter.write(getName(), Integer.toString(value), (double)v.getCountAtValueIteratedTo());
+ }
+ }
+ }
+
+ /**
+ * This is called periodically from the StatusThread. There's a single
+ * StatusThread per Client process. We optionally serialize the interval to
+ * log on this opportunity.
+ *
+ * @see site.ycsb.measurements.OneMeasurement#getSummary()
+ */
+ @Override
+ public String getSummary() {
+ Histogram intervalHistogram = getIntervalHistogramAndAccumulate();
+ // we use the summary interval as the histogram file interval.
+ if (histogramLogWriter != null) {
+ histogramLogWriter.outputIntervalHistogram(intervalHistogram);
+ }
+
+ DecimalFormat d = new DecimalFormat("#.##");
+ return "[" + getName() + ": Count=" + intervalHistogram.getTotalCount() + ", Max="
+ + intervalHistogram.getMaxValue() + ", Min=" + intervalHistogram.getMinValue() + ", Avg="
+ + d.format(intervalHistogram.getMean()) + ", 90=" + d.format(intervalHistogram.getValueAtPercentile(90))
+ + ", 99=" + d.format(intervalHistogram.getValueAtPercentile(99)) + ", 99.9="
+ + d.format(intervalHistogram.getValueAtPercentile(99.9)) + ", 99.99="
+ + d.format(intervalHistogram.getValueAtPercentile(99.99)) + "]";
+ }
+
+ private Histogram getIntervalHistogramAndAccumulate() {
+ Histogram intervalHistogram = histogram.getIntervalHistogram();
+ // add this to the total time histogram.
+ if (totalHistogram == null) {
+ totalHistogram = intervalHistogram;
+ } else {
+ totalHistogram.add(intervalHistogram);
+ }
+ return intervalHistogram;
+ }
+
+ /**
+ * Helper method to parse the given percentile value string.
+ *
+ * @param percentileString - comma delimited string of Integer values
+ * @return An Integer List of percentile values
+ */
+ private List getPercentileValues(String percentileString) {
+ List percentileValues = new ArrayList<>();
+
+ try {
+ for (String rawPercentile : percentileString.split(",")) {
+ percentileValues.add(Double.parseDouble(rawPercentile));
+ }
+ } catch (Exception e) {
+ // If the given hdrhistogram.percentiles value is unreadable for whatever reason,
+ // then calculate and return the default set.
+ System.err.println("[WARN] Couldn't read " + PERCENTILES_PROPERTY + " value: '" + percentileString +
+ "', the default of '" + PERCENTILES_PROPERTY_DEFAULT + "' will be used.");
+ e.printStackTrace();
+ return getPercentileValues(PERCENTILES_PROPERTY_DEFAULT);
+ }
+
+ return percentileValues;
+ }
+
+ /**
+ * Helper method to find the ordinal of any number. eg 1 -> 1st
+ * @param i number
+ * @return ordinal string
+ */
+ private String ordinal(Double i) {
+ String[] suffixes = new String[]{"th", "st", "nd", "rd", "th", "th", "th", "th", "th", "th"};
+ Integer j = i.intValue();
+ if (i % 1 == 0) {
+ switch (j % 100) {
+ case 11:
+ case 12:
+ case 13:
+ return j + "th";
+ default:
+ return j + suffixes[j % 10];
+ }
+ } else {
+ return i.toString();
+ }
+ }
+}
diff --git a/core/src/main/java/site/ycsb/measurements/OneMeasurementHistogram.java b/core/src/main/java/site/ycsb/measurements/OneMeasurementHistogram.java
new file mode 100644
index 0000000..01ec402
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/OneMeasurementHistogram.java
@@ -0,0 +1,166 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.measurements;
+
+import site.ycsb.measurements.exporter.MeasurementsExporter;
+
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.Properties;
+
+/**
+ * Take measurements and maintain a histogram of a given metric, such as READ LATENCY.
+ *
+ */
+public class OneMeasurementHistogram extends OneMeasurement {
+ public static final String BUCKETS = "histogram.buckets";
+ public static final String BUCKETS_DEFAULT = "1000";
+ public static final String VERBOSE_PROPERTY = "measurement.histogram.verbose";
+
+ /**
+ * Specify the range of latencies to track in the histogram.
+ */
+ private final int buckets;
+
+ /**
+ * Groups operations in discrete blocks of 1ms width.
+ */
+ private long[] histogram;
+
+ /**
+ * Counts all operations outside the histogram's range.
+ */
+ private long histogramoverflow;
+
+ /**
+ * The total number of reported operations.
+ */
+ private long operations;
+
+ /**
+ * The sum of each latency measurement over all operations.
+ * Calculated in ms.
+ */
+ private long totallatency;
+
+ /**
+ * The sum of each latency measurement squared over all operations.
+ * Used to calculate variance of latency.
+ * Calculated in ms.
+ */
+ private double totalsquaredlatency;
+
+ /**
+ * Whether or not to emit the histogram buckets.
+ */
+ private final boolean verbose;
+
+ //keep a windowed version of these stats for printing status
+ private long windowoperations;
+ private long windowtotallatency;
+
+ private int min;
+ private int max;
+
+ public OneMeasurementHistogram(String name, Properties props) {
+ super(name);
+ buckets = Integer.parseInt(props.getProperty(BUCKETS, BUCKETS_DEFAULT));
+ verbose = Boolean.valueOf(props.getProperty(VERBOSE_PROPERTY, String.valueOf(false)));
+ histogram = new long[buckets];
+ histogramoverflow = 0;
+ operations = 0;
+ totallatency = 0;
+ totalsquaredlatency = 0;
+ windowoperations = 0;
+ windowtotallatency = 0;
+ min = -1;
+ max = -1;
+ }
+
+ /* (non-Javadoc)
+ * @see site.ycsb.OneMeasurement#measure(int)
+ */
+ public synchronized void measure(int latency) {
+ //latency reported in us and collected in bucket by ms.
+ if (latency / 1000 >= buckets) {
+ histogramoverflow++;
+ } else {
+ histogram[latency / 1000]++;
+ }
+ operations++;
+ totallatency += latency;
+ totalsquaredlatency += ((double) latency) * ((double) latency);
+ windowoperations++;
+ windowtotallatency += latency;
+
+ if ((min < 0) || (latency < min)) {
+ min = latency;
+ }
+
+ if ((max < 0) || (latency > max)) {
+ max = latency;
+ }
+ }
+
+ @Override
+ public void exportMeasurements(MeasurementsExporter exporter) throws IOException {
+ double mean = totallatency / ((double) operations);
+ double variance = totalsquaredlatency / ((double) operations) - (mean * mean);
+ exporter.write(getName(), "Operations", operations);
+ exporter.write(getName(), "AverageLatency(us)", mean);
+ exporter.write(getName(), "LatencyVariance(us)", variance);
+ exporter.write(getName(), "MinLatency(us)", min);
+ exporter.write(getName(), "MaxLatency(us)", max);
+
+ long opcounter=0;
+ boolean done95th = false;
+ for (int i = 0; i < buckets; i++) {
+ opcounter += histogram[i];
+ if ((!done95th) && (((double) opcounter) / ((double) operations) >= 0.95)) {
+ exporter.write(getName(), "95thPercentileLatency(us)", i * 1000);
+ done95th = true;
+ }
+ if (((double) opcounter) / ((double) operations) >= 0.99) {
+ exporter.write(getName(), "99thPercentileLatency(us)", i * 1000);
+ break;
+ }
+ }
+
+ exportStatusCounts(exporter);
+
+ if (verbose) {
+ for (int i = 0; i < buckets; i++) {
+ exporter.write(getName(), Integer.toString(i), histogram[i]);
+ }
+
+ exporter.write(getName(), ">" + buckets, histogramoverflow);
+ }
+ }
+
+ @Override
+ public String getSummary() {
+ if (windowoperations == 0) {
+ return "";
+ }
+ DecimalFormat d = new DecimalFormat("#.##");
+ double report = ((double) windowtotallatency) / ((double) windowoperations);
+ windowtotallatency = 0;
+ windowoperations = 0;
+ return "[" + getName() + " AverageLatency(us)=" + d.format(report) + "]";
+ }
+}
diff --git a/core/src/main/java/site/ycsb/measurements/OneMeasurementRaw.java b/core/src/main/java/site/ycsb/measurements/OneMeasurementRaw.java
new file mode 100644
index 0000000..a3cc570
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/OneMeasurementRaw.java
@@ -0,0 +1,207 @@
+/**
+ * Copyright (c) 2015-2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.measurements;
+
+import site.ycsb.measurements.exporter.MeasurementsExporter;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.Properties;
+
+/**
+ * Record a series of measurements as raw data points without down sampling,
+ * optionally write to an output file when configured.
+ *
+ */
+public class OneMeasurementRaw extends OneMeasurement {
+ /**
+ * One raw data point, two fields: timestamp (ms) when the datapoint is
+ * inserted, and the value.
+ */
+ class RawDataPoint {
+ private final long timestamp;
+ private final int value;
+
+ public RawDataPoint(int value) {
+ this.timestamp = System.currentTimeMillis();
+ this.value = value;
+ }
+
+ public long timeStamp() {
+ return timestamp;
+ }
+
+ public int value() {
+ return value;
+ }
+ }
+
+ class RawDataPointComparator implements Comparator {
+ @Override
+ public int compare(RawDataPoint p1, RawDataPoint p2) {
+ if (p1.value() < p2.value()) {
+ return -1;
+ } else if (p1.value() == p2.value()) {
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+ }
+
+ /**
+ * Optionally, user can configure an output file to save the raw data points.
+ * Default is none, raw results will be written to stdout.
+ *
+ */
+ public static final String OUTPUT_FILE_PATH = "measurement.raw.output_file";
+ public static final String OUTPUT_FILE_PATH_DEFAULT = "";
+
+ /**
+ * Optionally, user can request to not output summary stats. This is useful
+ * if the user chains the raw measurement type behind the HdrHistogram type
+ * which already outputs summary stats. But even in that case, the user may
+ * still want this class to compute summary stats for them, especially if
+ * they want accurate computation of percentiles (because percentils computed
+ * by histogram classes are still approximations).
+ */
+ public static final String NO_SUMMARY_STATS = "measurement.raw.no_summary";
+ public static final String NO_SUMMARY_STATS_DEFAULT = "false";
+
+ private final PrintStream outputStream;
+
+ private boolean noSummaryStats = false;
+
+ private LinkedList measurements;
+ private long totalLatency = 0;
+
+ // A window of stats to print summary for at the next getSummary() call.
+ // It's supposed to be a one line summary, so we will just print count and
+ // average.
+ private int windowOperations = 0;
+ private long windowTotalLatency = 0;
+
+ public OneMeasurementRaw(String name, Properties props) {
+ super(name);
+
+ String outputFilePath = props.getProperty(OUTPUT_FILE_PATH, OUTPUT_FILE_PATH_DEFAULT);
+ if (!outputFilePath.isEmpty()) {
+ System.out.println("Raw data measurement: will output to result file: " +
+ outputFilePath);
+
+ try {
+ outputStream = new PrintStream(
+ new FileOutputStream(outputFilePath, true),
+ true);
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException("Failed to open raw data output file", e);
+ }
+
+ } else {
+ System.out.println("Raw data measurement: will output to stdout.");
+ outputStream = System.out;
+
+ }
+
+ noSummaryStats = Boolean.parseBoolean(props.getProperty(NO_SUMMARY_STATS,
+ NO_SUMMARY_STATS_DEFAULT));
+
+ measurements = new LinkedList<>();
+ }
+
+ @Override
+ public synchronized void measure(int latency) {
+ totalLatency += latency;
+ windowTotalLatency += latency;
+ windowOperations++;
+
+ measurements.add(new RawDataPoint(latency));
+ }
+
+ @Override
+ public void exportMeasurements(MeasurementsExporter exporter)
+ throws IOException {
+ // Output raw data points first then print out a summary of percentiles to
+ // stdout.
+
+ outputStream.println(getName() +
+ " latency raw data: op, timestamp(ms), latency(us)");
+ for (RawDataPoint point : measurements) {
+ outputStream.println(
+ String.format("%s,%d,%d", getName(), point.timeStamp(),
+ point.value()));
+ }
+ if (outputStream != System.out) {
+ outputStream.close();
+ }
+
+ int totalOps = measurements.size();
+ exporter.write(getName(), "Total Operations", totalOps);
+ if (totalOps > 0 && !noSummaryStats) {
+ exporter.write(getName(),
+ "Below is a summary of latency in microseconds:", -1);
+ exporter.write(getName(), "Average",
+ (double) totalLatency / (double) totalOps);
+
+ Collections.sort(measurements, new RawDataPointComparator());
+
+ exporter.write(getName(), "Min", measurements.get(0).value());
+ exporter.write(
+ getName(), "Max", measurements.get(totalOps - 1).value());
+ exporter.write(
+ getName(), "p1", measurements.get((int) (totalOps * 0.01)).value());
+ exporter.write(
+ getName(), "p5", measurements.get((int) (totalOps * 0.05)).value());
+ exporter.write(
+ getName(), "p50", measurements.get((int) (totalOps * 0.5)).value());
+ exporter.write(
+ getName(), "p90", measurements.get((int) (totalOps * 0.9)).value());
+ exporter.write(
+ getName(), "p95", measurements.get((int) (totalOps * 0.95)).value());
+ exporter.write(
+ getName(), "p99", measurements.get((int) (totalOps * 0.99)).value());
+ exporter.write(getName(), "p99.9",
+ measurements.get((int) (totalOps * 0.999)).value());
+ exporter.write(getName(), "p99.99",
+ measurements.get((int) (totalOps * 0.9999)).value());
+ }
+
+ exportStatusCounts(exporter);
+ }
+
+ @Override
+ public synchronized String getSummary() {
+ if (windowOperations == 0) {
+ return "";
+ }
+
+ String toReturn = String.format("%s count: %d, average latency(us): %.2f",
+ getName(), windowOperations,
+ (double) windowTotalLatency / (double) windowOperations);
+
+ windowTotalLatency = 0;
+ windowOperations = 0;
+
+ return toReturn;
+ }
+}
diff --git a/core/src/main/java/site/ycsb/measurements/OneMeasurementTimeSeries.java b/core/src/main/java/site/ycsb/measurements/OneMeasurementTimeSeries.java
new file mode 100644
index 0000000..03c3cf2
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/OneMeasurementTimeSeries.java
@@ -0,0 +1,146 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.measurements;
+
+import site.ycsb.measurements.exporter.MeasurementsExporter;
+
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.Properties;
+import java.util.Vector;
+
+class SeriesUnit {
+ /**
+ * @param time
+ * @param average
+ */
+ public SeriesUnit(long time, double average) {
+ this.time = time;
+ this.average = average;
+ }
+
+ protected final long time;
+ protected final double average;
+}
+
+/**
+ * A time series measurement of a metric, such as READ LATENCY.
+ */
+public class OneMeasurementTimeSeries extends OneMeasurement {
+
+ /**
+ * Granularity for time series; measurements will be averaged in chunks of this granularity. Units are milliseconds.
+ */
+ public static final String GRANULARITY = "timeseries.granularity";
+ public static final String GRANULARITY_DEFAULT = "1000";
+
+ private final int granularity;
+ private final Vector measurements;
+
+ private long start = -1;
+ private long currentunit = -1;
+ private long count = 0;
+ private long sum = 0;
+ private long operations = 0;
+ private long totallatency = 0;
+
+ //keep a windowed version of these stats for printing status
+ private int windowoperations = 0;
+ private long windowtotallatency = 0;
+
+ private int min = -1;
+ private int max = -1;
+
+ public OneMeasurementTimeSeries(String name, Properties props) {
+ super(name);
+ granularity = Integer.parseInt(props.getProperty(GRANULARITY, GRANULARITY_DEFAULT));
+ measurements = new Vector<>();
+ }
+
+ private synchronized void checkEndOfUnit(boolean forceend) {
+ long now = System.currentTimeMillis();
+
+ if (start < 0) {
+ currentunit = 0;
+ start = now;
+ }
+
+ long unit = ((now - start) / granularity) * granularity;
+
+ if ((unit > currentunit) || (forceend)) {
+ double avg = ((double) sum) / ((double) count);
+ measurements.add(new SeriesUnit(currentunit, avg));
+
+ currentunit = unit;
+
+ count = 0;
+ sum = 0;
+ }
+ }
+
+ @Override
+ public void measure(int latency) {
+ checkEndOfUnit(false);
+
+ count++;
+ sum += latency;
+ totallatency += latency;
+ operations++;
+ windowoperations++;
+ windowtotallatency += latency;
+
+ if (latency > max) {
+ max = latency;
+ }
+
+ if ((latency < min) || (min < 0)) {
+ min = latency;
+ }
+ }
+
+
+ @Override
+ public void exportMeasurements(MeasurementsExporter exporter) throws IOException {
+ checkEndOfUnit(true);
+
+ exporter.write(getName(), "Operations", operations);
+ exporter.write(getName(), "AverageLatency(us)", (((double) totallatency) / ((double) operations)));
+ exporter.write(getName(), "MinLatency(us)", min);
+ exporter.write(getName(), "MaxLatency(us)", max);
+
+ // TODO: 95th and 99th percentile latency
+
+ exportStatusCounts(exporter);
+ for (SeriesUnit unit : measurements) {
+ exporter.write(getName(), Long.toString(unit.time), unit.average);
+ }
+ }
+
+ @Override
+ public String getSummary() {
+ if (windowoperations == 0) {
+ return "";
+ }
+ DecimalFormat d = new DecimalFormat("#.##");
+ double report = ((double) windowtotallatency) / ((double) windowoperations);
+ windowtotallatency = 0;
+ windowoperations = 0;
+ return "[" + getName() + " AverageLatency(us)=" + d.format(report) + "]";
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/measurements/TwoInOneMeasurement.java b/core/src/main/java/site/ycsb/measurements/TwoInOneMeasurement.java
new file mode 100644
index 0000000..9ba53c1
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/TwoInOneMeasurement.java
@@ -0,0 +1,76 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.measurements;
+
+import site.ycsb.Status;
+import site.ycsb.measurements.exporter.MeasurementsExporter;
+
+import java.io.IOException;
+
+/**
+ * delegates to 2 measurement instances.
+ */
+public class TwoInOneMeasurement extends OneMeasurement {
+
+ private final OneMeasurement thing1, thing2;
+
+ public TwoInOneMeasurement(String name, OneMeasurement thing1, OneMeasurement thing2) {
+ super(name);
+ this.thing1 = thing1;
+ this.thing2 = thing2;
+ }
+
+ /**
+ * No need for synchronization, using CHM to deal with that.
+ */
+ @Override
+ public void reportStatus(final Status status) {
+ thing1.reportStatus(status);
+ }
+
+ /**
+ * It appears latency is reported in micros.
+ * Using {@link org.HdrHistogram.Recorder} to support concurrent updates to histogram.
+ */
+ @Override
+ public void measure(int latencyInMicros) {
+ thing1.measure(latencyInMicros);
+ thing2.measure(latencyInMicros);
+ }
+
+ /**
+ * This is called from a main thread, on orderly termination.
+ */
+ @Override
+ public void exportMeasurements(MeasurementsExporter exporter) throws IOException {
+ thing1.exportMeasurements(exporter);
+ thing2.exportMeasurements(exporter);
+ }
+
+ /**
+ * This is called periodically from the StatusThread. There's a single StatusThread per Client process.
+ * We optionally serialize the interval to log on this opportunity.
+ *
+ * @see site.ycsb.measurements.OneMeasurement#getSummary()
+ */
+ @Override
+ public String getSummary() {
+ return thing1.getSummary() + "\n" + thing2.getSummary();
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/measurements/exporter/JSONArrayMeasurementsExporter.java b/core/src/main/java/site/ycsb/measurements/exporter/JSONArrayMeasurementsExporter.java
new file mode 100644
index 0000000..6c70164
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/exporter/JSONArrayMeasurementsExporter.java
@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) 2015-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.measurements.exporter;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
+import org.codehaus.jackson.util.DefaultPrettyPrinter;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+/**
+ * Export measurements into a machine readable JSON Array of measurement objects.
+ */
+public class JSONArrayMeasurementsExporter implements MeasurementsExporter {
+ private final JsonFactory factory = new JsonFactory();
+ private JsonGenerator g;
+
+ public JSONArrayMeasurementsExporter(OutputStream os) throws IOException {
+ BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os));
+ g = factory.createJsonGenerator(bw);
+ g.setPrettyPrinter(new DefaultPrettyPrinter());
+ g.writeStartArray();
+ }
+
+ public void write(String metric, String measurement, int i) throws IOException {
+ g.writeStartObject();
+ g.writeStringField("metric", metric);
+ g.writeStringField("measurement", measurement);
+ g.writeNumberField("value", i);
+ g.writeEndObject();
+ }
+
+ public void write(String metric, String measurement, long i) throws IOException {
+ g.writeStartObject();
+ g.writeStringField("metric", metric);
+ g.writeStringField("measurement", measurement);
+ g.writeNumberField("value", i);
+ g.writeEndObject();
+ }
+
+ public void write(String metric, String measurement, double d) throws IOException {
+ g.writeStartObject();
+ g.writeStringField("metric", metric);
+ g.writeStringField("measurement", measurement);
+ g.writeNumberField("value", d);
+ g.writeEndObject();
+ }
+
+ public void close() throws IOException {
+ if (g != null) {
+ g.writeEndArray();
+ g.close();
+ }
+ }
+}
diff --git a/core/src/main/java/site/ycsb/measurements/exporter/JSONMeasurementsExporter.java b/core/src/main/java/site/ycsb/measurements/exporter/JSONMeasurementsExporter.java
new file mode 100644
index 0000000..c85f56b
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/exporter/JSONMeasurementsExporter.java
@@ -0,0 +1,73 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.measurements.exporter;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
+import org.codehaus.jackson.util.DefaultPrettyPrinter;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+/**
+ * Export measurements into a machine readable JSON file.
+ */
+public class JSONMeasurementsExporter implements MeasurementsExporter {
+
+ private final JsonFactory factory = new JsonFactory();
+ private JsonGenerator g;
+
+ public JSONMeasurementsExporter(OutputStream os) throws IOException {
+
+ BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os));
+ g = factory.createJsonGenerator(bw);
+ g.setPrettyPrinter(new DefaultPrettyPrinter());
+ }
+
+ public void write(String metric, String measurement, int i) throws IOException {
+ g.writeStartObject();
+ g.writeStringField("metric", metric);
+ g.writeStringField("measurement", measurement);
+ g.writeNumberField("value", i);
+ g.writeEndObject();
+ }
+
+ public void write(String metric, String measurement, long i) throws IOException {
+ g.writeStartObject();
+ g.writeStringField("metric", metric);
+ g.writeStringField("measurement", measurement);
+ g.writeNumberField("value", i);
+ g.writeEndObject();
+ }
+
+ public void write(String metric, String measurement, double d) throws IOException {
+ g.writeStartObject();
+ g.writeStringField("metric", metric);
+ g.writeStringField("measurement", measurement);
+ g.writeNumberField("value", d);
+ g.writeEndObject();
+ }
+
+ public void close() throws IOException {
+ if (g != null) {
+ g.close();
+ }
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/measurements/exporter/MeasurementsExporter.java b/core/src/main/java/site/ycsb/measurements/exporter/MeasurementsExporter.java
new file mode 100644
index 0000000..f3626e1
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/exporter/MeasurementsExporter.java
@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.measurements.exporter;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Used to export the collected measurements into a useful format, for example
+ * human readable text or machine readable JSON.
+ */
+public interface MeasurementsExporter extends Closeable {
+ /**
+ * Write a measurement to the exported format.
+ *
+ * @param metric Metric name, for example "READ LATENCY".
+ * @param measurement Measurement name, for example "Average latency".
+ * @param i Measurement to write.
+ * @throws IOException if writing failed
+ */
+ void write(String metric, String measurement, int i) throws IOException;
+
+ /**
+ * Write a measurement to the exported format.
+ *
+ * @param metric Metric name, for example "READ LATENCY".
+ * @param measurement Measurement name, for example "Average latency".
+ * @param i Measurement to write.
+ * @throws IOException if writing failed
+ */
+ void write(String metric, String measurement, long i) throws IOException;
+
+ /**
+ * Write a measurement to the exported format.
+ *
+ * @param metric Metric name, for example "READ LATENCY".
+ * @param measurement Measurement name, for example "Average latency".
+ * @param d Measurement to write.
+ * @throws IOException if writing failed
+ */
+ void write(String metric, String measurement, double d) throws IOException;
+}
diff --git a/core/src/main/java/site/ycsb/measurements/exporter/TextMeasurementsExporter.java b/core/src/main/java/site/ycsb/measurements/exporter/TextMeasurementsExporter.java
new file mode 100644
index 0000000..36acd92
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/exporter/TextMeasurementsExporter.java
@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.measurements.exporter;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+/**
+ * Write human readable text. Tries to emulate the previous print report method.
+ */
+public class TextMeasurementsExporter implements MeasurementsExporter {
+ private final BufferedWriter bw;
+
+ public TextMeasurementsExporter(OutputStream os) {
+ this.bw = new BufferedWriter(new OutputStreamWriter(os));
+ }
+
+ public void write(String metric, String measurement, int i) throws IOException {
+ bw.write("[" + metric + "], " + measurement + ", " + i);
+ bw.newLine();
+ }
+
+ public void write(String metric, String measurement, long i) throws IOException {
+ bw.write("[" + metric + "], " + measurement + ", " + i);
+ bw.newLine();
+ }
+
+ public void write(String metric, String measurement, double d) throws IOException {
+ bw.write("[" + metric + "], " + measurement + ", " + d);
+ bw.newLine();
+ }
+
+ public void close() throws IOException {
+ this.bw.close();
+ }
+}
diff --git a/core/src/main/java/site/ycsb/measurements/exporter/package-info.java b/core/src/main/java/site/ycsb/measurements/exporter/package-info.java
new file mode 100644
index 0000000..fe74b68
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/exporter/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+/**
+ * The YCSB measurements.exporter package.
+ */
+package site.ycsb.measurements.exporter;
+
diff --git a/core/src/main/java/site/ycsb/measurements/package-info.java b/core/src/main/java/site/ycsb/measurements/package-info.java
new file mode 100644
index 0000000..78fc129
--- /dev/null
+++ b/core/src/main/java/site/ycsb/measurements/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+/**
+ * The YCSB measurements package.
+ */
+package site.ycsb.measurements;
+
diff --git a/core/src/main/java/site/ycsb/package-info.java b/core/src/main/java/site/ycsb/package-info.java
new file mode 100644
index 0000000..276715d
--- /dev/null
+++ b/core/src/main/java/site/ycsb/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2015 - 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+/**
+ * The YCSB core package.
+ */
+package site.ycsb;
+
diff --git a/core/src/main/java/site/ycsb/workloads/ConstantOccupancyWorkload.java b/core/src/main/java/site/ycsb/workloads/ConstantOccupancyWorkload.java
new file mode 100644
index 0000000..2403bb8
--- /dev/null
+++ b/core/src/main/java/site/ycsb/workloads/ConstantOccupancyWorkload.java
@@ -0,0 +1,90 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ * Copyrihght (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package site.ycsb.workloads;
+
+import site.ycsb.Client;
+import site.ycsb.WorkloadException;
+import site.ycsb.generator.NumberGenerator;
+import site.ycsb.workloads.core.CoreHelper;
+
+import java.util.Properties;
+import static site.ycsb.workloads.core.CoreConstants.*;
+/**
+ * A disk-fragmenting workload.
+ *
+ * Properties to control the client:
+ *
+ *
+ *
disksize: how many bytes of storage can the disk store? (default 100,000,000)
+ *
occupancy: what fraction of the available storage should be used? (default 0.9)
+ *
requestdistribution: what distribution should be used to select the records to operate on - uniform,
+ * zipfian or latest (default: histogram)
+ *
+ */
+public class ConstantOccupancyWorkload extends CoreWorkload {
+ private long disksize;
+ private long storageages;
+ private double occupancy;
+
+ private long objectCount;
+
+ public static final String STORAGE_AGE_PROPERTY = "storageages";
+ public static final long STORAGE_AGE_PROPERTY_DEFAULT = 10;
+
+ public static final String DISK_SIZE_PROPERTY = "disksize";
+ public static final long DISK_SIZE_PROPERTY_DEFAULT = 100 * 1000 * 1000;
+
+ public static final String OCCUPANCY_PROPERTY = "occupancy";
+ public static final double OCCUPANCY_PROPERTY_DEFAULT = 0.9;
+
+ @Override
+ public void init(Properties p) throws WorkloadException {
+ disksize = Long.parseLong(p.getProperty(DISK_SIZE_PROPERTY, String.valueOf(DISK_SIZE_PROPERTY_DEFAULT)));
+ storageages = Long.parseLong(p.getProperty(STORAGE_AGE_PROPERTY, String.valueOf(STORAGE_AGE_PROPERTY_DEFAULT)));
+ occupancy = Double.parseDouble(p.getProperty(OCCUPANCY_PROPERTY, String.valueOf(OCCUPANCY_PROPERTY_DEFAULT)));
+
+ if (p.getProperty(Client.RECORD_COUNT_PROPERTY) != null ||
+ p.getProperty(Client.INSERT_COUNT_PROPERTY) != null ||
+ p.getProperty(Client.OPERATION_COUNT_PROPERTY) != null) {
+ System.err.println("Warning: record, insert or operation count was set prior to initting " +
+ "ConstantOccupancyWorkload. Overriding old values.");
+ }
+ NumberGenerator g = CoreHelper.getFieldLengthGenerator(p);
+ double fieldsize = g.mean();
+ int fieldcount = Integer.parseInt(p.getProperty(FIELD_COUNT_PROPERTY, FIELD_COUNT_PROPERTY_DEFAULT));
+
+ objectCount = (long) (occupancy * (disksize / (fieldsize * fieldcount)));
+ if (objectCount == 0) {
+ throw new IllegalStateException("Object count was zero. Perhaps disksize is too low?");
+ }
+ p.setProperty(Client.RECORD_COUNT_PROPERTY, String.valueOf(objectCount));
+ p.setProperty(Client.OPERATION_COUNT_PROPERTY, String.valueOf(storageages * objectCount));
+ p.setProperty(Client.INSERT_COUNT_PROPERTY, String.valueOf(objectCount));
+
+ super.init(p);
+ }
+
+}
diff --git a/core/src/main/java/site/ycsb/workloads/CoreWorkload.java b/core/src/main/java/site/ycsb/workloads/CoreWorkload.java
new file mode 100644
index 0000000..c36d4b3
--- /dev/null
+++ b/core/src/main/java/site/ycsb/workloads/CoreWorkload.java
@@ -0,0 +1,574 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc., Copyright (c) 2016-2020 YCSB contributors. All rights reserved.
+ * Copyright (c) 2023 - 2024 benchANT GmbH. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package site.ycsb.workloads;
+
+import site.ycsb.*;
+import site.ycsb.generator.*;
+import site.ycsb.generator.acknowledge.AcknowledgedCounterGenerator;
+import site.ycsb.measurements.Measurements;
+import site.ycsb.workloads.core.CoreHelper;
+import site.ycsb.wrappers.DatabaseField;
+import site.ycsb.wrappers.Wrappers;
+import static site.ycsb.workloads.core.CoreConstants.*;
+
+import java.util.*;
+
+/**
+ * The core benchmark scenario. Represents a set of clients doing simple CRUD operations. The
+ * relative proportion of different kinds of operations, and other properties of the workload,
+ * are controlled by parameters specified at runtime.
+ *
+ * Properties to control the client:
+ *
+ *
fieldcount: the number of fields in a record (default: 10)
+ *
fieldlength: the size of each field (default: 100)
+ *
minfieldlength: the minimum size of each field (default: 1)
+ *
readallfields: should reads read all fields (true) or just one (false) (default: true)
+ *
writeallfields: should updates and read/modify/writes update all fields (true) or just
+ * one (false) (default: false)
+ *
readproportion: what proportion of operations should be reads (default: 0.95)
+ *
updateproportion: what proportion of operations should be updates (default: 0.05)
+ *
insertproportion: what proportion of operations should be inserts (default: 0)
+ *
scanproportion: what proportion of operations should be scans (default: 0)
+ *
readmodifywriteproportion: what proportion of operations should be read a record,
+ * modify it, write it back (default: 0)
+ *
requestdistribution: what distribution should be used to select the records to operate
+ * on - uniform, zipfian, hotspot, sequential, exponential or latest (default: uniform)
+ *
minscanlength: for scans, what is the minimum number of records to scan (default: 1)
+ *
maxscanlength: for scans, what is the maximum number of records to scan (default: 1000)
+ *
scanlengthdistribution: for scans, what distribution should be used to choose the
+ * number of records to scan, for each scan, between 1 and maxscanlength (default: uniform)
+ *
insertstart: for parallel loads and runs, defines the starting record for this
+ * YCSB instance (default: 0)
+ *
insertcount: for parallel loads and runs, defines the number of records for this
+ * YCSB instance (default: recordcount)
+ *
zeropadding: for generating a record sequence compatible with string sort order by
+ * 0 padding the record number. Controls the number of 0s to use for padding. (default: 1)
+ * For example for row 5, with zeropadding=1 you get 'user5' key and with zeropading=8 you get
+ * 'user00000005' key. In order to see its impact, zeropadding needs to be bigger than number of
+ * digits in the record number.
+ *
insertorder: should records be inserted in order by key ("ordered"), or in hashed
+ * order ("hashed") (default: hashed)
+ *
fieldnameprefix: what should be a prefix for field names, the shorter may decrease the
+ * required storage size (default: "field")
+ *
+ */
+public class CoreWorkload extends Workload {
+
+ protected String table;
+ protected List fieldnames;
+
+ /**
+ * Generator object that produces field lengths. The value of this depends on the properties that
+ * start with "FIELD_LENGTH_".
+ */
+ protected NumberGenerator fieldlengthgenerator;
+
+ protected boolean readallfields;
+
+ protected boolean readallfieldsbyname;
+
+ protected boolean writeallfields;
+
+ /**
+ * Set to true if want to check correctness of reads. Must also
+ * be set to true during loading phase to function.
+ */
+ private boolean dataintegrity;
+
+
+ protected NumberGenerator keysequence;
+ protected DiscreteGenerator operationchooser;
+ protected NumberGenerator keychooser;
+ protected NumberGenerator fieldchooser;
+ protected AcknowledgedCounterGenerator transactioninsertkeysequence;
+ protected NumberGenerator scanlength;
+ protected boolean orderedinserts;
+ protected long fieldcount;
+ protected long recordcount;
+ protected int zeropadding;
+ protected int insertionRetryLimit;
+ protected int insertionRetryInterval;
+
+ private Measurements measurements = Measurements.getMeasurements();
+
+ public static String buildKeyName(long keynum, int zeropadding, boolean orderedinserts) {
+ if (!orderedinserts) {
+ keynum = Utils.hash(keynum);
+ }
+ String value = Long.toString(keynum);
+ int fill = zeropadding - value.length();
+ String prekey = "user";
+ for (int i = 0; i < fill; i++) {
+ prekey += '0';
+ }
+ return prekey + value;
+ }
+
+
+
+ /**
+ * Initialize the scenario.
+ * Called once, in the main client thread, before any operations are started.
+ */
+ @Override
+ public void init(Properties p) throws WorkloadException {
+ table = p.getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT);
+
+ fieldcount =
+ Long.parseLong(p.getProperty(FIELD_COUNT_PROPERTY, FIELD_COUNT_PROPERTY_DEFAULT));
+ final String fieldnameprefix = p.getProperty(FIELD_NAME_PREFIX, FIELD_NAME_PREFIX_DEFAULT);
+ fieldnames = new ArrayList<>();
+ for (int i = 0; i < fieldcount; i++) {
+ fieldnames.add(fieldnameprefix + i);
+ }
+ fieldlengthgenerator = CoreHelper.getFieldLengthGenerator(p);
+
+ recordcount =
+ Long.parseLong(p.getProperty(Client.RECORD_COUNT_PROPERTY, Client.DEFAULT_RECORD_COUNT));
+ if (recordcount == 0) {
+ recordcount = Integer.MAX_VALUE;
+ }
+ String requestdistrib =
+ p.getProperty(REQUEST_DISTRIBUTION_PROPERTY, REQUEST_DISTRIBUTION_PROPERTY_DEFAULT);
+ int minscanlength =
+ Integer.parseInt(p.getProperty(MIN_SCAN_LENGTH_PROPERTY, MIN_SCAN_LENGTH_PROPERTY_DEFAULT));
+ int maxscanlength =
+ Integer.parseInt(p.getProperty(MAX_SCAN_LENGTH_PROPERTY, MAX_SCAN_LENGTH_PROPERTY_DEFAULT));
+ String scanlengthdistrib =
+ p.getProperty(SCAN_LENGTH_DISTRIBUTION_PROPERTY, SCAN_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT);
+
+ long insertstart =
+ Long.parseLong(p.getProperty(INSERT_START_PROPERTY, INSERT_START_PROPERTY_DEFAULT));
+ long insertcount= Long.parseLong(p.getProperty(INSERT_COUNT_PROPERTY, String.valueOf(recordcount - insertstart)));
+ // Confirm valid values for insertstart and insertcount in relation to recordcount
+ if (recordcount < (insertstart + insertcount)) {
+ System.err.println("Invalid combination of insertstart, insertcount and recordcount.");
+ System.err.println("recordcount must be bigger than insertstart + insertcount.");
+ System.exit(-1);
+ }
+ zeropadding =
+ Integer.parseInt(p.getProperty(ZERO_PADDING_PROPERTY, ZERO_PADDING_PROPERTY_DEFAULT));
+
+ readallfields = Boolean.parseBoolean(
+ p.getProperty(READ_ALL_FIELDS_PROPERTY, READ_ALL_FIELDS_PROPERTY_DEFAULT));
+ readallfieldsbyname = Boolean.parseBoolean(
+ p.getProperty(READ_ALL_FIELDS_BY_NAME_PROPERTY, READ_ALL_FIELDS_BY_NAME_PROPERTY_DEFAULT));
+ writeallfields = Boolean.parseBoolean(
+ p.getProperty(WRITE_ALL_FIELDS_PROPERTY, WRITE_ALL_FIELDS_PROPERTY_DEFAULT));
+
+ dataintegrity = Boolean.parseBoolean(
+ p.getProperty(DATA_INTEGRITY_PROPERTY, DATA_INTEGRITY_PROPERTY_DEFAULT));
+ // Confirm that fieldlengthgenerator returns a constant if data
+ // integrity check requested.
+ if (dataintegrity && !(p.getProperty(
+ FIELD_LENGTH_DISTRIBUTION_PROPERTY,
+ FIELD_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT)).equals("constant")) {
+ System.err.println("Must have constant field size to check data integrity.");
+ System.exit(-1);
+ }
+ if (dataintegrity) {
+ System.out.println("Data integrity is enabled.");
+ }
+
+ if (p.getProperty(INSERT_ORDER_PROPERTY, INSERT_ORDER_PROPERTY_DEFAULT).compareTo("hashed") == 0) {
+ orderedinserts = false;
+ } else {
+ orderedinserts = true;
+ }
+
+ keysequence = new CounterGenerator(insertstart);
+ operationchooser = CoreHelper.createOperationGenerator(p);
+
+ transactioninsertkeysequence = CoreHelper.createTransactionInsertKeyGenerator(p, recordcount);
+ if (requestdistrib.compareTo("uniform") == 0) {
+ keychooser = new UniformLongGenerator(insertstart, insertstart + insertcount - 1);
+ } else if (requestdistrib.compareTo("exponential") == 0) {
+ double percentile = Double.parseDouble(p.getProperty(
+ ExponentialGenerator.EXPONENTIAL_PERCENTILE_PROPERTY,
+ ExponentialGenerator.EXPONENTIAL_PERCENTILE_DEFAULT));
+ double frac = Double.parseDouble(p.getProperty(
+ ExponentialGenerator.EXPONENTIAL_FRAC_PROPERTY,
+ ExponentialGenerator.EXPONENTIAL_FRAC_DEFAULT));
+ keychooser = new ExponentialGenerator(percentile, recordcount * frac);
+ } else if (requestdistrib.compareTo("sequential") == 0) {
+ keychooser = new SequentialGenerator(insertstart, insertstart + insertcount - 1);
+ } else if (requestdistrib.compareTo("zipfian") == 0) {
+ // it does this by generating a random "next key" in part by taking the modulus over the
+ // number of keys.
+ // If the number of keys changes, this would shift the modulus, and we don't want that to
+ // change which keys are popular so we'll actually construct the scrambled zipfian generator
+ // with a keyspace that is larger than exists at the beginning of the test. that is, we'll predict
+ // the number of inserts, and tell the scrambled zipfian generator the number of existing keys
+ // plus the number of predicted keys as the total keyspace. then, if the generator picks a key
+ // that hasn't been inserted yet, will just ignore it and pick another key. this way, the size of
+ // the keyspace doesn't change from the perspective of the scrambled zipfian generator
+ final double insertproportion = Double.parseDouble(
+ p.getProperty(INSERT_PROPORTION_PROPERTY, INSERT_PROPORTION_PROPERTY_DEFAULT));
+ long opcount = Long.parseLong(p.getProperty(Client.OPERATION_COUNT_PROPERTY));
+ long expectednewkeys = (long) ((opcount) * insertproportion * 2.0); // 2 is fudge factor
+
+ keychooser = new ScrambledZipfianGenerator(insertstart, insertstart + insertcount + expectednewkeys);
+ } else if (requestdistrib.compareTo("latest") == 0) {
+ keychooser = new SkewedLatestGenerator(transactioninsertkeysequence);
+ } else if (requestdistrib.equals("hotspot")) {
+ double hotsetfraction =
+ Double.parseDouble(p.getProperty(HOTSPOT_DATA_FRACTION, HOTSPOT_DATA_FRACTION_DEFAULT));
+ double hotopnfraction =
+ Double.parseDouble(p.getProperty(HOTSPOT_OPN_FRACTION, HOTSPOT_OPN_FRACTION_DEFAULT));
+ keychooser = new HotspotIntegerGenerator(insertstart, insertstart + insertcount - 1,
+ hotsetfraction, hotopnfraction);
+ } else {
+ throw new WorkloadException("Unknown request distribution \"" + requestdistrib + "\"");
+ }
+
+ fieldchooser = new UniformLongGenerator(0, fieldcount - 1);
+
+ if (scanlengthdistrib.compareTo("uniform") == 0) {
+ scanlength = new UniformLongGenerator(minscanlength, maxscanlength);
+ } else if (scanlengthdistrib.compareTo("zipfian") == 0) {
+ scanlength = new ZipfianGenerator(minscanlength, maxscanlength);
+ } else {
+ throw new WorkloadException(
+ "Distribution \"" + scanlengthdistrib + "\" not allowed for scan length");
+ }
+
+ insertionRetryLimit = Integer.parseInt(p.getProperty(
+ INSERTION_RETRY_LIMIT, INSERTION_RETRY_LIMIT_DEFAULT));
+ insertionRetryInterval = Integer.parseInt(p.getProperty(
+ INSERTION_RETRY_INTERVAL, INSERTION_RETRY_INTERVAL_DEFAULT));
+ }
+
+ /**
+ * Builds a value for a randomly chosen field.
+ */
+ protected HashMap buildSingleValue(String key) {
+ HashMap value = new HashMap<>();
+
+ String fieldkey = fieldnames.get(fieldchooser.nextValue().intValue());
+ ByteIterator data;
+ if (dataintegrity) {
+ data = new StringByteIterator(buildDeterministicValue(key, fieldkey));
+ } else {
+ // fill with random data
+ data = new RandomByteIterator(fieldlengthgenerator.nextValue().longValue());
+ }
+ value.put(fieldkey, data);
+
+ return value;
+ }
+
+ protected HashMap oldBuildValues(String key) {
+ HashMap values = new HashMap<>();
+ for (String fieldkey : fieldnames) {
+ ByteIterator data;
+ if (dataintegrity) {
+ data = new StringByteIterator(buildDeterministicValue(key, fieldkey));
+ } else {
+ // fill with random data
+ data = new RandomByteIterator(fieldlengthgenerator.nextValue().longValue());
+ }
+ values.put(fieldkey, data);
+ }
+ return values;
+ }
+
+ /**
+ * Builds values for all fields.
+ */
+ protected List buildValues(String key) {
+ List values = new ArrayList<>();
+ // HashMap values = new HashMap<>();
+
+ for (String fieldkey : fieldnames) {
+ ByteIterator data;
+ if (dataintegrity) {
+ data = new StringByteIterator(buildDeterministicValue(key, fieldkey));
+ } else {
+ // fill with random data
+ data = new RandomByteIterator(fieldlengthgenerator.nextValue().longValue());
+ }
+ values.add(new DatabaseField(fieldkey, Wrappers.wrapIterator(data)));
+ }
+ return values;
+ }
+
+ /**
+ * Build a deterministic value given the key information.
+ */
+ private String buildDeterministicValue(String key, String fieldkey) {
+ int size = fieldlengthgenerator.nextValue().intValue();
+ StringBuilder sb = new StringBuilder(size);
+ sb.append(key);
+ sb.append(':');
+ sb.append(fieldkey);
+ while (sb.length() < size) {
+ sb.append(':');
+ sb.append(sb.toString().hashCode());
+ }
+ sb.setLength(size);
+
+ return sb.toString();
+ }
+
+ /**
+ * Do one insert operation. Because it will be called concurrently from multiple client threads,
+ * this function must be thread safe. However, avoid synchronized, or the threads will block waiting
+ * for each other, and it will be difficult to reach the target throughput. Ideally, this function would
+ * have no side effects other than DB operations.
+ */
+ @Override
+ public boolean doInsert(DB db, Object threadstate) {
+ long keynum = keysequence.nextValue().longValue();
+ String dbkey = CoreWorkload.buildKeyName(keynum, zeropadding, orderedinserts);
+ List values = buildValues(dbkey);
+
+ Status status;
+ int numOfRetries = 0;
+ do {
+ status = db.insert(table, dbkey, values);
+ if (null != status && status.isOk()) {
+ break;
+ }
+ // Retry if configured. Without retrying, the load process will fail
+ // even if one single insertion fails. User can optionally configure
+ // an insertion retry limit (default is 0) to enable retry.
+ if (++numOfRetries <= insertionRetryLimit) {
+ System.err.println("Retrying insertion, retry count: " + numOfRetries);
+ try {
+ // Sleep for a random number between [0.8, 1.2)*insertionRetryInterval.
+ int sleepTime = (int) (1000 * insertionRetryInterval * (0.8 + 0.4 * Math.random()));
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ break;
+ }
+
+ } else {
+ System.err.println("Error inserting, not retrying any more. number of attempts: " + numOfRetries +
+ "Insertion Retry Limit: " + insertionRetryLimit);
+ break;
+
+ }
+ } while (true);
+
+ return null != status && status.isOk();
+ }
+
+ /**
+ * Do one transaction operation. Because it will be called concurrently from multiple client
+ * threads, this function must be thread safe. However, avoid synchronized, or the threads will block waiting
+ * for each other, and it will be difficult to reach the target throughput. Ideally, this function would
+ * have no side effects other than DB operations.
+ */
+ @Override
+ public boolean doTransaction(DB db, Object threadstate) {
+ String operation = operationchooser.nextString();
+ if(operation == null) {
+ return false;
+ }
+
+ switch (operation) {
+ case "READ":
+ doTransactionRead(db);
+ break;
+ case "UPDATE":
+ doTransactionUpdate(db);
+ break;
+ case "INSERT":
+ doTransactionInsert(db);
+ break;
+ case "SCAN":
+ doTransactionScan(db);
+ break;
+ default:
+ doTransactionReadModifyWrite(db);
+ }
+
+ return true;
+ }
+
+ /**
+ * Results are reported in the first three buckets of the histogram under
+ * the label "VERIFY".
+ * Bucket 0 means the expected data was returned.
+ * Bucket 1 means incorrect data was returned.
+ * Bucket 2 means null data was returned when some data was expected.
+ */
+ protected void verifyRow(String key, HashMap cells) {
+ Status verifyStatus = Status.OK;
+ long startTime = System.nanoTime();
+ if (!cells.isEmpty()) {
+ for (Map.Entry entry : cells.entrySet()) {
+ if (!entry.getValue().toString().equals(buildDeterministicValue(key, entry.getKey()))) {
+ verifyStatus = Status.UNEXPECTED_STATE;
+ break;
+ }
+ }
+ } else {
+ // This assumes that null data is never valid
+ verifyStatus = Status.ERROR;
+ }
+ long endTime = System.nanoTime();
+ measurements.measure("VERIFY", (int) (endTime - startTime) / 1000);
+ measurements.reportStatus("VERIFY", verifyStatus);
+ }
+
+ protected long nextKeynum() {
+ long keynum;
+ if (keychooser instanceof ExponentialGenerator) {
+ do {
+ keynum = transactioninsertkeysequence.lastValue() - keychooser.nextValue().longValue();
+ } while (keynum < 0);
+ } else {
+ do {
+ keynum = keychooser.nextValue().longValue();
+ } while (keynum > transactioninsertkeysequence.lastValue());
+ }
+ return keynum;
+ }
+
+ public void doTransactionRead(DB db) {
+ // choose a random key
+ long keynum = nextKeynum();
+
+ String keyname = CoreWorkload.buildKeyName(keynum, zeropadding, orderedinserts);
+
+ HashSet fields = null;
+
+ if (!readallfields) {
+ // read a random field
+ String fieldname = fieldnames.get(fieldchooser.nextValue().intValue());
+
+ fields = new HashSet();
+ fields.add(fieldname);
+ } else if (dataintegrity || readallfieldsbyname) {
+ // pass the full field list if dataintegrity is on for verification
+ fields = new HashSet(fieldnames);
+ }
+
+ HashMap