diff --git a/.travis.yml b/.travis.yml index f719ce86..236ed3b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ jobs: - rm -rf $HOME/build32 && cp -r $PWD $HOME/build32 - rm -rf $HOME/build64 && cp -r $PWD $HOME/build64 - make -C $HOME/build32 - - XLEN=64 RISCV_TOOLCHAIN_PATH=$TOOLDIR/riscv64-gnu-toolchain make -C $HOME/build64 + - XLEN=64 make -C $HOME/build64 - stage: test name: unittest script: cp -r $HOME/build32 build && cd build && ./ci/travis_run.py ./ci/regression.sh --unittest @@ -47,13 +47,13 @@ jobs: script: cp -r $HOME/build32 build && cd build && ./ci/travis_run.py ./ci/regression.sh --isa - stage: test name: isa64 - script: cp -r $HOME/build64 build && cd build && XLEN=64 RISCV_TOOLCHAIN_PATH=$TOOLDIR/riscv64-gnu-toolchain ./ci/travis_run.py ./ci/regression.sh --isa + script: cp -r $HOME/build64 build && cd build && XLEN=64 ./ci/travis_run.py ./ci/regression.sh --isa - stage: test name: regression script: cp -r $HOME/build32 build && cd build && ./ci/travis_run.py ./ci/regression.sh --regression - stage: test name: regression64 - script: cp -r $HOME/build64 build && cd build && XLEN=64 RISCV_TOOLCHAIN_PATH=$TOOLDIR/riscv64-gnu-toolchain ./ci/travis_run.py ./ci/regression.sh --regression + script: cp -r $HOME/build64 build && cd build && XLEN=64 ./ci/travis_run.py ./ci/regression.sh --regression - stage: test name: opencl script: cp -r $HOME/build32 build && cd build && ./ci/travis_run.py ./ci/regression.sh --opencl diff --git a/README.md b/README.md index 34343a28..0db8e2ea 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Vortex is a full-stack open-source RISC-V GPGPU. - `miscs`: Miscellaneous resources. ## Build Instructions +More detailed build instructions can be found [here](docs/install_vortex.md). ### Supported OS Platforms - Ubuntu 18.04 - Centos 7 diff --git a/ci/blackbox.sh b/ci/blackbox.sh index 5e06ae65..d5deacec 100755 --- a/ci/blackbox.sh +++ b/ci/blackbox.sh @@ -16,7 +16,17 @@ show_usage() { echo "Vortex BlackBox Test Driver v1.0" - echo "Usage: $0 [[--clusters=#n] [--cores=#n] [--warps=#n] [--threads=#n] [--l2cache] [--l3cache] [[--driver=#name] [--app=#app] [--args=#args] [--debug=#level] [--scope] [--perf=#class] [--rebuild=0|1] [--log=logfile] [--help]]" + echo "Usage: $0 [[--clusters=#n] [--cores=#n] [--warps=#n] [--threads=#n] [--l2cache] [--l3cache] [[--driver=#name] [--app=#app] [--args=#args] [--debug=#level] [--scope] [--perf=#class] [--rebuild=#n] [--log=logfile] [--help]]" +} + +show_help() +{ + show_usage + echo " where" + echo "--driver: simx, rtlsim, oape, xrt" + echo "--app: any subfolder test under regression or opencl" + echo "--class: 0=disable, 1=pipeline, 2=memsys" + echo "--rebuild: 0=disable, 1=force, 2=auto, 3=temp" } SCRIPT_DIR=$(dirname "$0") @@ -36,6 +46,7 @@ SCOPE=0 HAS_ARGS=0 PERF_CLASS=0 REBUILD=2 +TEMPBUILD=0 LOGFILE=run.log for i in "$@" @@ -102,7 +113,7 @@ case $i in shift ;; --help) - show_usage + show_help exit 0 ;; *) @@ -112,6 +123,12 @@ case $i in esac done +if [ $REBUILD -eq 3 ]; +then + REBUILD=1 + TEMPBUILD=1 +fi + case $DRIVER in simx) DRIVER_PATH=$VORTEX_HOME/runtime/simx @@ -174,53 +191,119 @@ make -C $VORTEX_HOME/runtime/stub > /dev/null if [ $DEBUG -ne 0 ] then - # driver initialization - if [ $SCOPE -eq 1 ] - then - echo "running: DEBUG=$DEBUG_LEVEL SCOPE=1 CONFIGS="$CONFIGS" make -C $DRIVER_PATH" - DEBUG=$DEBUG_LEVEL SCOPE=1 CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null - else - echo "running: DEBUG=$DEBUG_LEVEL CONFIGS="$CONFIGS" make -C $DRIVER_PATH" - DEBUG=$DEBUG_LEVEL CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null - fi - # running application - if [ $HAS_ARGS -eq 1 ] + if [ $TEMPBUILD -eq 1 ] then - echo "running: OPTS=$ARGS make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1" - OPTS=$ARGS make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1 - status=$? + # setup temp directory + TEMPDIR=$(mktemp -d) + mkdir -p "$TEMPDIR/$DRIVER" + + # driver initialization + if [ $SCOPE -eq 1 ] + then + echo "running: DESTDIR=$TEMPDIR/$DRIVER DEBUG=$DEBUG_LEVEL SCOPE=1 CONFIGS=$CONFIGS make -C $DRIVER_PATH" + DESTDIR="$TEMPDIR/$DRIVER" DEBUG=$DEBUG_LEVEL SCOPE=1 CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null + else + echo "running: DESTDIR=$TEMPDIR/$DRIVER DEBUG=$DEBUG_LEVEL CONFIGS=$CONFIGS make -C $DRIVER_PATH" + DESTDIR="$TEMPDIR/$DRIVER" DEBUG=$DEBUG_LEVEL CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null + fi + + # running application + if [ $HAS_ARGS -eq 1 ] + then + echo "running: VORTEX_RT_PATH=$TEMPDIR OPTS=$ARGS make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1" + VORTEX_RT_PATH=$TEMPDIR OPTS=$ARGS make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1 + status=$? + else + echo "running: VORTEX_RT_PATH=$TEMPDIR make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1" + VORTEX_RT_PATH=$TEMPDIR make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1 + status=$? + fi + + # cleanup temp directory + trap "rm -rf $TEMPDIR" EXIT else - echo "running: make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1" - make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1 - status=$? + # driver initialization + if [ $SCOPE -eq 1 ] + then + echo "running: DEBUG=$DEBUG_LEVEL SCOPE=1 CONFIGS=$CONFIGS make -C $DRIVER_PATH" + DEBUG=$DEBUG_LEVEL SCOPE=1 CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null + else + echo "running: DEBUG=$DEBUG_LEVEL CONFIGS=$CONFIGS make -C $DRIVER_PATH" + DEBUG=$DEBUG_LEVEL CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null + fi + + # running application + if [ $HAS_ARGS -eq 1 ] + then + echo "running: OPTS=$ARGS make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1" + OPTS=$ARGS make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1 + status=$? + else + echo "running: make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1" + make -C $APP_PATH run-$DRIVER > $LOGFILE 2>&1 + status=$? + fi fi if [ -f "$APP_PATH/trace.vcd" ] then mv -f $APP_PATH/trace.vcd . fi -else - # driver initialization - if [ $SCOPE -eq 1 ] +else + if [ $TEMPBUILD -eq 1 ] then - echo "running: SCOPE=1 CONFIGS="$CONFIGS" make -C $DRIVER_PATH" - SCOPE=1 CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null + # setup temp directory + TEMPDIR=$(mktemp -d) + mkdir -p "$TEMPDIR/$DRIVER" + + # driver initialization + if [ $SCOPE -eq 1 ] + then + echo "running: DESTDIR=$TEMPDIR/$DRIVER SCOPE=1 CONFIGS=$CONFIGS make -C $DRIVER_PATH" + DESTDIR="$TEMPDIR/$DRIVER" SCOPE=1 CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null + else + echo "running: DESTDIR=$TEMPDIR/$DRIVER CONFIGS=$CONFIGS make -C $DRIVER_PATH" + DESTDIR="$TEMPDIR/$DRIVER" CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null + fi + + # running application + if [ $HAS_ARGS -eq 1 ] + then + echo "running: VORTEX_RT_PATH=$TEMPDIR OPTS=$ARGS make -C $APP_PATH run-$DRIVER" + VORTEX_RT_PATH=$TEMPDIR OPTS=$ARGS make -C $APP_PATH run-$DRIVER + status=$? + else + echo "running: VORTEX_RT_PATH=$TEMPDIR make -C $APP_PATH run-$DRIVER" + VORTEX_RT_PATH=$TEMPDIR make -C $APP_PATH run-$DRIVER + status=$? + fi + + # cleanup temp directory + trap "rm -rf $TEMPDIR" EXIT else - echo "running: CONFIGS="$CONFIGS" make -C $DRIVER_PATH" - CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null - fi - - # running application - if [ $HAS_ARGS -eq 1 ] - then - echo "running: OPTS=$ARGS make -C $APP_PATH run-$DRIVER" - OPTS=$ARGS make -C $APP_PATH run-$DRIVER - status=$? - else - echo "running: make -C $APP_PATH run-$DRIVER" - make -C $APP_PATH run-$DRIVER - status=$? + + # driver initialization + if [ $SCOPE -eq 1 ] + then + echo "running: SCOPE=1 CONFIGS=$CONFIGS make -C $DRIVER_PATH" + SCOPE=1 CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null + else + echo "running: CONFIGS=$CONFIGS make -C $DRIVER_PATH" + CONFIGS="$CONFIGS" make -C $DRIVER_PATH > /dev/null + fi + + # running application + if [ $HAS_ARGS -eq 1 ] + then + echo "running: OPTS=$ARGS make -C $APP_PATH run-$DRIVER" + OPTS=$ARGS make -C $APP_PATH run-$DRIVER + status=$? + else + echo "running: make -C $APP_PATH run-$DRIVER" + make -C $APP_PATH run-$DRIVER + status=$? + fi fi fi diff --git a/ci/toolchain_env.sh b/ci/toolchain_env.sh index 4046a6a7..440a899e 100644 --- a/ci/toolchain_env.sh +++ b/ci/toolchain_env.sh @@ -16,14 +16,11 @@ TOOLDIR=${TOOLDIR:=/opt} -export RISCV_TOOLCHAIN_PATH=$TOOLDIR/riscv-gnu-toolchain -export LLVM_POCL=$TOOLDIR/llvm-pocl -export LLVM_VORTEX=$TOOLDIR/llvm-vortex export VERILATOR_ROOT=$TOOLDIR/verilator export PATH=$VERILATOR_ROOT/bin:$PATH + export SV2V_PATH=$TOOLDIR/sv2v export PATH=$SV2V_PATH/bin:$PATH + export YOSYS_PATH=$TOOLDIR/yosys export PATH=$YOSYS_PATH/bin:$PATH -export POCL_CC_PATH=$TOOLDIR/pocl/compiler -export POCL_RT_PATH=$TOOLDIR/pocl/runtime diff --git a/docs/assets/img/cache_hierarchy.png b/docs/assets/img/cache_hierarchy.png deleted file mode 100644 index 876f5fe2..00000000 Binary files a/docs/assets/img/cache_hierarchy.png and /dev/null differ diff --git a/docs/assets/img/cache_microarchitecture.png b/docs/assets/img/cache_microarchitecture.png new file mode 100644 index 00000000..2214aa93 Binary files /dev/null and b/docs/assets/img/cache_microarchitecture.png differ diff --git a/docs/assets/img/vortex_bank.png b/docs/assets/img/vortex_bank.png deleted file mode 100644 index 8c10995e..00000000 Binary files a/docs/assets/img/vortex_bank.png and /dev/null differ diff --git a/docs/assets/img/vortex_cache_top_module.png b/docs/assets/img/vortex_cache_top_module.png deleted file mode 100644 index ecb8be98..00000000 Binary files a/docs/assets/img/vortex_cache_top_module.png and /dev/null differ diff --git a/docs/assets/img/vortex_microarchitecture.png b/docs/assets/img/vortex_microarchitecture.png new file mode 100644 index 00000000..7fde5296 Binary files /dev/null and b/docs/assets/img/vortex_microarchitecture.png differ diff --git a/docs/assets/img/vortex_microarchitecture_v2.png b/docs/assets/img/vortex_microarchitecture_v2.png deleted file mode 100644 index c0e85891..00000000 Binary files a/docs/assets/img/vortex_microarchitecture_v2.png and /dev/null differ diff --git a/docs/cache_subsystem.md b/docs/cache_subsystem.md index 4bfc0ded..9de309e1 100644 --- a/docs/cache_subsystem.md +++ b/docs/cache_subsystem.md @@ -2,69 +2,26 @@ The Vortex Cache Sub-system has the following main properties: -- High-bandwidth with bank parallelism -- Snoop protocol to flush data for CPU access -- Generic design: Dcache, Icache, Shared Memory, L2 cache, L3 cache +- High-bandwidth transfer with Multi-bank parallelism +- Non-blocking pipelined architecture with local MSHR +- Configurable design: Dcache, Icache, L2 cache, L3 cache -### Cache Hierarchy +### Cache Microarchitecture -![Image of Cache Hierarchy](./assets/img/cache_hierarchy.png) +![Image of Cache Hierarchy](./assets/img/cache_microarchitecture.png) -- Cache can be configured to be any level in the hierarchy -- Caches communicate via snooping -- Cache flush from AFU is passed down the hierarchy​ +The Vortex cache is comprised of multiple parallel banks. It is comprised of the following modules: +- **Bank request dispatch crossbar**: assign a bank to incoming requests and resolve collision using stalls. +- **Bank response merge crossbar**: merge result from banks and forward to the core response. +- **Memory request multiplexer**: arbitrate bank memory requests +- **Memory response demultiplexer**: forward memory response to the corresponding bank. +- **Flush Unit**: perform tag memory initialization. -### VX_cache.v (Top Module) +Incoming requests entering the cache are sent to a dispatch crossbar that select the corresponding bank for each request, resolving bank collisions with stalls. The result output of each bank is merge back into outgoing response port via merger crossbar. Each bank intergates a non-blocking pipeline with a local Miss Status Holding Register (MSHR) to reduce the miss rate. The bank pipeline consists of the following stages: -VX.cache.v is the top module of the cache verilog code located in the `/hw/rtl/cache` directory. +- **Schedule**: Selects the next request into the pipeline from the incoming core request, memory fill, or the MSHR entry, with priority given to the latter. +- **Tag Access**: A single-port read/write access to the tag store. +- **Data Access**: Single-port read/write access to the data store. +- **Response Handling**: Core response back to the core. -![Image of Vortex Cache](./assets/img/vortex_cache_top_module.png) - -- Configurable (Cache size, number of banks, bank line size, etc.) -- I/O signals - - Core Request - - Core Rsp - - DRAM Req - - DRAM Rsp - - Snoop Rsp - - Snoop Rsp - - Snoop Forwarding Out - - Snoop Forwarding In -- Bank Select - - Assigns valid and ready signals for each bank -- Snoop Forwarder -- DRAM Request Arbiter - - Prepares cache response for communication with DRAM -- Snoop Response Arbiter - - Sends snoop response -- Core Response Merge - - Cache accesses one line at a time. As a result, each request may not come back in the same response. This module tries to recombine the responses by thread ID. - -### VX_cache_bank.v - -VX_cache_bank.v is the verilog code that handles cache bank functionality and is located in the `/hw/rtl/cache` directory. - -![Image of Vortex Cache Bank](./assets/img/vortex_bank.png) - -- Allows for high throughput​ -- Each bank contains queues to hold requests to the cache​ -- I/O signals - - Core request​ - - Core Response​ - - DRAM Fill Requests​ - - DRAM Fill Response​ - - DRAM WB Requests​ - - Snp Request​ - - Snp Response -- Request Priority: DRAM fill, miss reserve, core request, snoop request​ -- Snoop Request Queue​ -- DRAM Fill Queue​ -- Core Req Arbiter​ - - Requests to be processed by the bank -- Tag Data Store​ - - Registers for valid, dirty, dirtyb, tag, and data​ - - Length of registers determined by lines in the bank​ -- Tag Data Access:​ - - I/O: stall, snoop info, force request miss - - Writes to cache or sends read response; hit or miss determined here - - A missed request goes to the miss reserve if it is not a snoop request or DRAM fill \ No newline at end of file +Deadlocks inside the cache can occur when the MSHR is full and a new request is already in the pipeline. It can also occur when the memory request queue is full, and there is an incoming memory response. The cache mitigates MSHR deadlocks by using an early full signal before a new request is issued and similarly mitigates memory deadlocks by ensuring that its request queue never fills up. diff --git a/docs/continuous_integration.md b/docs/continuous_integration.md new file mode 100644 index 00000000..abb90580 --- /dev/null +++ b/docs/continuous_integration.md @@ -0,0 +1,36 @@ +# Continuous Integration +- Each time you push to the repo, the Continuous Integration pipeline will run +- This pipeline consists of creating the correct development environment, building your code, and running all tests +- This is an extensive pipeline so it might take some time to complete + + +## Protecting Master Branch +Navigate to your Repository: +Open your repository on GitHub. + +Click on "Settings": +In the upper-right corner of your repository page, click on the "Settings" tab. + +Select "Branches" in the left sidebar: +On the left sidebar, look for the "Branches" option and click on it. + +Choose the Branch: +Under "Branch protection rules," select the branch you want to protect. In this case, choose the main branch. + +Enable Branch Protection:`` +Check the box that says "Protect this branch." + +Configure Protection Settings: +You can configure various protection settings. Some common settings include: + +Require pull request reviews before merging: This ensures that changes are reviewed before being merged. +Require status checks to pass before merging: This ensures that automated tests and checks are passing. +Require signed commits: This enforces that commits are signed with a verified signature. +Restrict Who Can Push: +You can further restrict who can push directly to the branch. You might want to limit this privilege to specific people or teams. + +Save Changes: +Once you've configured the protection settings, scroll down and click on the "Save changes" button. + +Now, your main branch is protected, and certain criteria must be met before changes can be pushed directly to it. Contributors will need to create pull requests, have their changes reviewed, and meet other specified criteria before the changes can be merged into the main branch. + diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 00000000..14e0ccd0 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,18 @@ +# Contributing to Vortex on Github + +## Github Details +- There are two main repos, `vortex` (public, this one) and `vortex-dev` (private) +- todo: Most current development is on `vortex` +- If you have a legacy version of `vortex`, you can use the releases branch or tags to access the repo at that point in time + +## Contribution Process +- You should create a new branch from develop that is clearly named with the feature that you want to add +- Avoid pushing directly to the `master` branch instead you will need to make a Pull Request (PR) +- There should be protections in place that prevent pushing directly to the main branch, but don't rely on it +- When you make a PR it will be tested against the continuous integration (ci) pipeline (see `continuous_integration.md`) +- It is not sufficient to just write some tests, they need to be incorporated into the ci pipeline to make sure they are run +- During a PR, you might receive feedback regarding your changes and you might need to make further commits to your branch + + +## Creating and Adding Tests +see `testing.md` \ No newline at end of file diff --git a/docs/environment_setup.md b/docs/environment_setup.md new file mode 100644 index 00000000..a55060ee --- /dev/null +++ b/docs/environment_setup.md @@ -0,0 +1,45 @@ +# Environment Setup +These instructions apply to the development vortex repo using the updated toolchain. The updated toolchain is considered to be any commit of `master` pulled from July 2, 2023 onwards. The toolchain update in question can be viewed in this [commit](https://github.com/vortexgpgpu/vortex-dev/commit/0048496ba28d7b9a209a0e569d52d60f2b68fc04). Therefore, if you are unsure whether you are using the new toolchain or not, then you should check the `ci` folder for the existence of the `toolchain_prebuilt.sh` script. Furthermore, you should notice that the `toolchain_install.sh` script has the legacy `llvm()` split into `llvm-vortex()` and `llvm-pocl()`. + + +## Set Up on Your Own System +The toolchain binaries provided with Vortex are built on Ubuntu-based systems. To install Vortex on your own system, [follow these instructions](install_vortex.md). + + +## Servers for Georgia Tech Students and Collaborators +### Volvo +Volvo is a 64-core server provided by HPArch. You need valid credentials to access it. If you don't already have access, you can get in contact with your mentor to ask about setting your account up. + +Setup on Volvo: +1. Connect to Georgia Tech's VPN or ssh into another machine on campus +2. `ssh volvo.cc.gatech.edu` +3. Clone Vortex to your home directory: `git clone --recursive https://github.com/vortexgpgpu/vortex.git` +4. `source /nethome/software/set_vortex_env.sh` to set up the necessary environment variables. +5. `make -s` in the `vortex` root directory +6. Run a test program: `./ci/blackbox.sh --cores=2 --app=dogfood` + +### Nio +Nio is a 20-core desktop server provided by HPArch. If you have access to Volvo, you also have access to Nio. + +Setup on Nio: +1. Connect to Georgia Tech's VPN or ssh into another machine on campus +2. `ssh nio.cc.gatech.edu` +3. Clone Vortex to your home directory: `git clone --recursive https://github.com/vortexgpgpu/vortex.git` +4. `source /opt/set_vortex_env_dev.sh` to set up the necessary environment variables. +5. `make -s` in the `vortex` root directory +6. Run a test program: `./ci/blackbox.sh --cores=2 --app=dogfood` + + +## Docker (Experimental) +Docker allows for isolated pre-built environments to be created, shared and used. The emulation mode required for ARM-based processors will incur a decrease in performance. Currently, the dockerfile is not included with the official vortex repository and is not actively maintained or supported. + +### Setup with Docker +1. Clone repo recursively onto your local machine: `git clone --recursive https://github.com/vortexgpgpu/vortex.git` +2. Download the dockerfile from [here](https://github.gatech.edu/gist/usubramanya3/f1bf3e953faa38a6372e1292ffd0b65c) and place it in the root of the repo. +3. Build the Dockerfile into an image: `docker build --platform=linux/amd64 -t vortex -f dockerfile .` +4. Run a container based on the image: `docker run --rm -v ./:/root/vortex/ -it --name vtx-dev --privileged=true --platform=linux/amd64 vortex` +5. Install the toolchain `./ci/toolchain_install.sh --all` (once per container) +6. `make -s` in `vortex` root directory +7. Run a test program: `./ci/blackbox.sh --cores=2 --app=dogfood` + +You may exit from a container and resume a container you have exited or start a second terminal session `docker exec -it bash` diff --git a/docs/fpga_setup.md b/docs/fpga_setup.md index 61ff481f..88e0c3c0 100644 --- a/docs/fpga_setup.md +++ b/docs/fpga_setup.md @@ -9,9 +9,6 @@ OPAE Environment Setup $ export C_INCLUDE_PATH=$OPAE_HOME/include:$C_INCLUDE_PATH $ export LIBRARY_PATH=$OPAE_HOME/lib:$LIBRARY_PATH $ export LD_LIBRARY_PATH=$OPAE_HOME/lib:$LD_LIBRARY_PATH - $ export RISCV_TOOLCHAIN_PATH=/opt/riscv-gnu-toolchain - $ export PATH=:/opt/verilator/bin:$PATH - $ export VERILATOR_ROOT=/opt/verilator OPAE Build ------------------ diff --git a/docs/index.md b/docs/index.md index 19bbfa4d..07faa292 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,7 +13,8 @@ ## Installation -- Refer to the build instructions in [README](../README.md). +- For the different environments Vortex supports, [read this document](environment_setup.md). +- To install on your own system, [follow this document](install_vortex.md). ## Quick Start Scenarios @@ -28,4 +29,4 @@ Running Vortex simulators with different configurations: - Run dogfood driver test with simx driver and Vortex config of 4 cluster, 4 cores, 8 warps, 6 threads - $ ./ci/blackbox.sh --driver=simx --clusters=4 --cores=4 --warps=8 --threads=6 --app=dogfood \ No newline at end of file + $ ./ci/blackbox.sh --driver=simx --clusters=4 --cores=4 --warps=8 --threads=6 --app=dogfood diff --git a/docs/install_vortex.md b/docs/install_vortex.md new file mode 100644 index 00000000..80cdc426 --- /dev/null +++ b/docs/install_vortex.md @@ -0,0 +1,124 @@ +# Installing and Setting Up the Vortex Environment + +## Ubuntu 18.04, 20.04 + +1. Install the following dependencies: + + ``` + sudo apt-get install build-essential zlib1g-dev libtinfo-dev libncurses5 uuid-dev libboost-serialization-dev libpng-dev libhwloc-dev + ``` + +2. Upgrade gcc to 11: + + ``` + sudo apt-get install gcc-11 g++-11 + ``` + + Multiple gcc versions on Ubuntu can be managed with update-alternatives, e.g.: + + ``` + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 9 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 9 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 11 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 11 + ``` + +3. Download the Vortex codebase: + + ``` + git clone --recursive https://github.com/vortexgpgpu/vortex.git + ``` + +4. Install Vortex's prebuilt toolchain: + + ``` + cd vortex + sudo ./ci/toolchain_install.sh -all + + # By default, the toolchain will install to /opt folder. This is recommended, but you can install the toolchain to a different directory by setting DESTDIR. + DESTDIR=$TOOLDIR ./ci/toolchain_install.sh -all + ``` + +5. Set up environment: + + ``` + export VORTEX_HOME=$TOOLDIR/vortex + export LLVM_VORTEX=$TOOLDIR/llvm-vortex + export LLVM_POCL=$TOOLDIR/llvm-pocl + export POCL_CC_PATH=$TOOLDIR/pocl/compiler + export POCL_RT_PATH=$TOOLDIR/pocl/runtime + export RISCV_TOOLCHAIN_PATH=$TOOLDIR/riscv-gnu-toolchain + export VERILATOR_ROOT=$TOOLDIR/verilator + export SV2V_PATH=$TOOLDIR/sv2v + export YOSYS_PATH=$TOOLDIR/yosys + + export PATH=$YOSYS_PATH/bin:$SV2V_PATH/bin:$VERILATOR_ROOT/bin:$PATH + ``` + +6. Build Vortex + + ``` + make + ``` + + +## RHEL 8 +Note: depending on the system, some of the toolchain may need to be recompiled for non-Ubuntu Linux. The source for the tools can be found [here](https://github.com/vortexgpgpu/). + +1. Install the following dependencies: + + ``` + sudo yum install libpng-devel boost boost-devel boost-serialization libuuid-devel opencl-headers hwloc hwloc-devel gmp-devel compat-hwloc1 + ``` + +2. Upgrade gcc to 11: + + ``` + sudo yum install gcc-toolset-11 + ``` + + Multiple gcc versions on Red Hat can be managed with scl + +3. Install MPFR 4.2.0: + + Download [the source](https://ftp.gnu.org/gnu/mpfr/) and follow [the installation documentation](https://www.mpfr.org/mpfr-current/mpfr.html#How-to-Install). + +4. Download the Vortex codebase: + + ``` + git clone --recursive https://github.com/vortexgpgpu/vortex.git + ``` + +5. Install Vortex's prebuilt toolchain: + + ``` + cd vortex + sudo ./ci/toolchain_install.sh -all + + # By default, the toolchain will install to /opt folder. This is recommended, but you can install the toolchain to a different directory by setting DESTDIR. + DESTDIR=$TOOLDIR ./ci/toolchain_install.sh -all + ``` + +6. Set up environment: + + ``` + export VORTEX_HOME=$TOOLDIR/vortex + export LLVM_VORTEX=$TOOLDIR/llvm-vortex + export LLVM_POCL=$TOOLDIR/llvm-pocl + export POCL_CC_PATH=$TOOLDIR/pocl/compiler + export POCL_RT_PATH=$TOOLDIR/pocl/runtime + export RISCV_TOOLCHAIN_PATH=$TOOLDIR/riscv-gnu-toolchain + export VERILATOR_ROOT=$TOOLDIR/verilator + export SV2V_PATH=$TOOLDIR/sv2v + export YOSYS_PATH=$TOOLDIR/yosys + + export PATH=$YOSYS_PATH/bin:$SV2V_PATH/bin:$VERILATOR_ROOT/bin:$PATH + + export LD_LIBRARY_PATH=/src/.libs:$LD_LIBRARY_PATH + ``` + +7. Build Vortex + + ``` + make + ``` diff --git a/docs/microarchitecture.md b/docs/microarchitecture.md index 972da7b0..3459abcc 100644 --- a/docs/microarchitecture.md +++ b/docs/microarchitecture.md @@ -24,71 +24,57 @@ Vortex uses the SIMT (Single Instruction, Multiple Threads) execution model with - Control the number of warps to activate during execution - `WSPAWN` *count, addr*: activate count warps and jump to addr location - **Control-Flow Divergence** - - Control threads to activate when a branch diverges - - `SPLIT` *predicate*: apply 'taken' predicate thread mask adn save 'not-taken' into IPDOM stack - - `JOIN`: restore 'not-taken' thread mask + - Control threads activation when a branch diverges + - `SPLIT` *taken, predicate*: apply predicate thread mask and save current state into IPDOM stack + - `JOIN`: pop IPDOM stack to restore thread mask + - `PRED` *predicate, restore_mask*: thread predicate instruction - **Warp Synchronization** - `BAR` *id, count*: stall warps entering barrier *id* until count is reached ### Vortex Pipeline/Datapath -![Image of Vortex Microarchitecture](./assets/img/vortex_microarchitecture_v2.png) +![Image of Vortex Microarchitecture](./assets/img/vortex_microarchitecture.png) -Vortex has a 5-stage pipeline: FI | ID | Issue | EX | WB. +Vortex has a 6-stage pipeline: + +- **Schedule** + - Warp Scheduler + - Schedule the next PC into the pipeline + - Track stalled, active warps + - IPDOM Stack + - Save split/join states for divergent threads + - Inflight Tracker + - Track in-flight instructions - **Fetch** - - Warp Scheduler - - Track stalled & active warps, resolve branches and barriers, maintain split/join IPDOM stack - - Instruction Cache - - Retrieve instruction from cache, issue I-cache requests/responses + - Retrieve instructions from memory + - Handle I-cache requests/responses - **Decode** - - Decode fetched instructions, notify warp scheduler when the following instructions are decoded: - - Branch, tmc, split/join, wspawn - - Precompute used_regs mask (needed for Issue stage) + - Decode fetched instructions + - Notify warp scheduler on control instructions - **Issue** - - Scheduling - - In-order issue (operands/execute unit ready), out-of-order commit - IBuffer - - Store fetched instructions, separate queues per-warp, selects next warp through round-robin scheduling + - Store decoded instructions in separate per-warp queues - Scoreboard - Track in-use registers - - GPRs (General-Purpose Registers) stage - - Fetch issued instruction operands and send operands to execute unit + - Check register use for decoded instructions + - Operands Collector + - Fetch the operands for issued instructions from the register file - **Execute** - ALU Unit - - Single-cycle operations (+,-,>>,<<,&,|,^), Branch instructions (Share ALU resources) - - MULDIV Unit - - Multiplier - done in 2 cycles - - Divider - division and remainder, done in 32 cycles - - Implements serial alogrithm (Stalls the pipeline) + - Handle arithmetic and branch operations - FPU Unit - - Multi-cycle operations, uses `FPnew` Library on ASIC, uses hard DSPs on FPGA - - CSR Unit - - Store constant status registers - device caps, FPU status flags, performance counters - - Handle external CSR requests (requests from host CPU) + - Handle floating-point operations - LSU Unit - - Handle load/store operations, issue D-cache requests, handle D-cache responses - - Commit load responses - saves storage, Scoreboard tracks completion - - GPGPU Unit - - Handle GPGPU instructions - - TMC, WSPAWN, SPLIT, BAR - - JOIN is handled by Warp Scheduler (upon SPLIT response) + - Handle load/store operations + - SFU Unit + - Handle warp control operations + - Handle Control Status Registers (CSRs) operations - **Commit** - - Commit - - Update CSR flags, update performance counters - - Writeback - - Write result back to GPRs, notify Scoreboard (release in-use register), select candidate instruction (ALU unit has highest priority) -- **Clustering** - - Group mulitple cores into clusters (optionally share L2 cache) - - Group multiple clusters (optionally share L3 cache) - - Configurable at build time - - Default configuration: - - #Clusters = 1 - - #Cores = 4 - - #Warps = 4 - - #Threads = 4 -- **FPGA AFU Interface** - - Manage CPU-GPU comunication - - Query devices caps, load kernel instructions and resource buffers, start kernel execution, read destination buffers - - Local Memory - GPU access to local DRAM - - Reserved I/O addresses - redirect to host CPU, console output \ No newline at end of file + - Write result back to the register file and update the Scoreboard. + +### Vortex clustering architecture +- Sockets + - Grouping multiple cores sharing L1 cache +- Clusters + - Grouping of sockets sharing L2 cache \ No newline at end of file diff --git a/docs/testing.md b/docs/testing.md index 552008f3..0103e7bc 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2,18 +2,18 @@ ## Running a Vortex application -The framework provides a utility script: blakcbox.sh under the /ci/ folder for executing applications in the tests tree. +The framework provides a utility script: blackbox.sh under the /ci/ folder for executing applications in the tests tree. You can query the commandline options of the tool using: - $ ./ci/blakcbox.sh --help + $ ./ci/blackbox.sh --help To execute sgemm test program on the simx driver and passing "-n10" as argument to sgemm: - $ ./ci/blakcbox.sh --driver=simx --app=sgemm --args="-n10" + $ ./ci/blackbox.sh --driver=simx --app=sgemm --args="-n10" You can execute the same application of a GPU architecture with 2 cores: - $ ./ci/blakcbox.sh --core=2 --driver=simx --app=sgemm --args="-n10" + $ ./ci/blackbox.sh --core=2 --driver=simx --app=sgemm --args="-n10" When excuting, Blackbox needs to recompile the driver if the desired architecture changes. It tracks the latest configuration in a file under the current directory blackbox..cache. @@ -30,4 +30,18 @@ You can execute the default regression suite by running the following commands a You can execute the default opncl suite by running the following commands at the root folder. $ make -C tests/opencl run-simx - $ make -C tests/opencl run-rtlsim \ No newline at end of file + $ make -C tests/opencl run-rtlsim + +## Creating Your Own Regression Tests +- Inside `test/` you will find a series of folders which are named based on what they test +- You can view the tests to see which ones have tests similar to what you are trying to create new tests for +- once you have found a similar baseline, you can copy the folder and rename it to what you are planning to test +- `testcases.h` contains each of the test case templates +- `main.cpp` contains the implementation of each of the test cases and builds a test suite of all the tests cases you want + +Compile the test case: `make -C tests/regression// clean-all && make -C tests/regression//` + +Run the test case: `./ci/blackbox.sh --driver=simx --cores=4 --app= --debug` + +## Adding Your Tests to the CI Pipeline +see `continuous_integration.md` \ No newline at end of file diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index d537249d..02505171 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -43,7 +43,16 @@ module VX_cluster import VX_gpu_pkg::*; #( `ifdef SCOPE localparam scope_socket = 0; `SCOPE_IO_SWITCH (scope_socket + `NUM_SOCKETS); -`endif +`endif + +`ifdef PERF_ENABLE + VX_mem_perf_if mem_perf_tmp_if(); + assign mem_perf_tmp_if.icache = 'x; + assign mem_perf_tmp_if.dcache = 'x; + assign mem_perf_tmp_if.l3cache = mem_perf_if.l3cache; + assign mem_perf_tmp_if.smem = 'x; + assign mem_perf_tmp_if.mem = mem_perf_if.mem; +`endif `ifdef GBAR_ENABLE @@ -69,18 +78,7 @@ module VX_cluster import VX_gpu_pkg::*; #( .reset (gbar_reset), .gbar_bus_if (gbar_bus_if) ); -`endif -`ifdef PERF_ENABLE - VX_mem_perf_if mem_perf_tmp_if(); - cache_perf_t perf_l2cache; - - assign mem_perf_tmp_if.icache = 'x; - assign mem_perf_tmp_if.dcache = 'x; - assign mem_perf_tmp_if.l2cache = perf_l2cache; - assign mem_perf_tmp_if.l3cache = mem_perf_if.l3cache; - assign mem_perf_tmp_if.smem = 'x; - assign mem_perf_tmp_if.mem = mem_perf_if.mem; `endif VX_mem_bus_if #( @@ -102,7 +100,7 @@ module VX_cluster import VX_gpu_pkg::*; #( .MSHR_SIZE (`L2_MSHR_SIZE), .MRSQ_SIZE (`L2_MRSQ_SIZE), .MREQ_SIZE (`L2_MREQ_SIZE), - .TAG_WIDTH (L1_MEM_ARB_TAG_WIDTH), + .TAG_WIDTH (L2_TAG_WIDTH), .WRITE_ENABLE (1), .UUID_WIDTH (`UUID_WIDTH), .CORE_OUT_REG (2), @@ -113,7 +111,7 @@ module VX_cluster import VX_gpu_pkg::*; #( .clk (clk), .reset (l2_reset), `ifdef PERF_ENABLE - .cache_perf (perf_l2cache), + .cache_perf (mem_perf_tmp_if.l2cache), `endif .core_bus_if (per_socket_mem_bus_if), .mem_bus_if (mem_bus_if) @@ -146,6 +144,7 @@ module VX_cluster import VX_gpu_pkg::*; #( .SOCKET_ID ((CLUSTER_ID * `NUM_SOCKETS) + i) ) socket ( `SCOPE_IO_BIND (scope_socket+i) + .clk (clk), .reset (socket_reset), @@ -156,7 +155,7 @@ module VX_cluster import VX_gpu_pkg::*; #( .dcr_bus_if (socket_dcr_bus_if), .mem_bus_if (per_socket_mem_bus_if[i]), - + `ifdef GBAR_ENABLE .gbar_bus_if (per_socket_gbar_bus_if[i]), `endif @@ -167,6 +166,6 @@ module VX_cluster import VX_gpu_pkg::*; #( ); end - `BUFFER_BUSY (busy, (| per_socket_busy), (`NUM_SOCKETS > 1)); + `BUFFER_EX(busy, (| per_socket_busy), 1'b1, (`NUM_SOCKETS > 1)); endmodule diff --git a/hw/rtl/VX_config.vh b/hw/rtl/VX_config.vh index e0c462f5..056b8def 100644 --- a/hw/rtl/VX_config.vh +++ b/hw/rtl/VX_config.vh @@ -191,13 +191,21 @@ `define STALL_TIMEOUT (100000 * (1 ** (`L2_ENABLED + `L3_ENABLED))) `endif +`ifndef SV_DPI +`define DPI_DISABLE +`endif + `ifndef FPU_FPNEW `ifndef FPU_DSP `ifndef FPU_DPI -`ifdef SYNTHESIS -`define FPU_DSP -`else +`ifndef SYNTHESIS +`ifndef DPI_DISABLE `define FPU_DPI +`else +`define FPU_DSP +`endif +`else +`define FPU_DSP `endif `endif `endif @@ -223,18 +231,18 @@ // Number of ALU units `ifndef NUM_ALU_LANES -`define NUM_ALU_LANES `UP(`NUM_THREADS / 2) +`define NUM_ALU_LANES `NUM_THREADS `endif `ifndef NUM_ALU_BLOCKS -`define NUM_ALU_BLOCKS `UP(`ISSUE_WIDTH / 1) +`define NUM_ALU_BLOCKS `ISSUE_WIDTH `endif // Number of FPU units `ifndef NUM_FPU_LANES -`define NUM_FPU_LANES `UP(`NUM_THREADS / 2) +`define NUM_FPU_LANES `NUM_THREADS `endif `ifndef NUM_FPU_BLOCKS -`define NUM_FPU_BLOCKS `UP(`ISSUE_WIDTH / 1) +`define NUM_FPU_BLOCKS `ISSUE_WIDTH `endif // Number of LSU units @@ -258,7 +266,10 @@ `endif // LSU Duplicate Address Check -`ifdef LSU_DUP +`ifndef LSU_DUP_DISABLE +`define LSU_DUP_ENABLE +`endif +`ifdef LSU_DUP_ENABLE `define LSU_DUP_ENABLED 1 `else `define LSU_DUP_ENABLED 0 @@ -285,8 +296,8 @@ // Floating-Point Units /////////////////////////////////////////////////////// // Size of FPU Request Queue -`ifndef FPU_REQ_QUEUE_SIZE -`define FPU_REQ_QUEUE_SIZE (2 * (`NUM_THREADS / `NUM_FPU_LANES)) +`ifndef FPUQ_SIZE +`define FPUQ_SIZE (2 * (`NUM_THREADS / `NUM_FPU_LANES)) `endif // FNCP Latency @@ -377,7 +388,7 @@ // Number of Cache Units `ifndef NUM_ICACHES -`define NUM_ICACHES `UP(`NUM_CORES / 4) +`define NUM_ICACHES `UP(`SOCKET_SIZE / 4) `endif // Cache Size @@ -407,7 +418,7 @@ // Number of Associative Ways `ifndef ICACHE_NUM_WAYS -`define ICACHE_NUM_WAYS 2 +`define ICACHE_NUM_WAYS 1 `endif // Dcache Configurable Knobs ////////////////////////////////////////////////// @@ -426,7 +437,7 @@ // Number of Cache Units `ifndef NUM_DCACHES -`define NUM_DCACHES `UP(`NUM_CORES / 4) +`define NUM_DCACHES `UP(`SOCKET_SIZE / 4) `endif // Cache Size @@ -436,7 +447,7 @@ // Number of Banks `ifndef DCACHE_NUM_BANKS -`define DCACHE_NUM_BANKS (`NUM_LSU_LANES) +`define DCACHE_NUM_BANKS `MIN(`NUM_LSU_LANES, 4) `endif // Core Response Queue Size @@ -461,7 +472,7 @@ // Number of Associative Ways `ifndef DCACHE_NUM_WAYS -`define DCACHE_NUM_WAYS 2 +`define DCACHE_NUM_WAYS 1 `endif // SM Configurable Knobs ////////////////////////////////////////////////////// @@ -520,7 +531,7 @@ // Number of Associative Ways `ifndef L2_NUM_WAYS -`define L2_NUM_WAYS 4 +`define L2_NUM_WAYS 2 `endif // L3cache Configurable Knobs ///////////////////////////////////////////////// diff --git a/hw/rtl/VX_define.vh b/hw/rtl/VX_define.vh index 5dcf1f2f..63f2d42d 100644 --- a/hw/rtl/VX_define.vh +++ b/hw/rtl/VX_define.vh @@ -1,418 +1,437 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`ifndef VX_DEFINE_VH -`define VX_DEFINE_VH - -`include "VX_platform.vh" -`include "VX_config.vh" -`include "VX_types.vh" - -/////////////////////////////////////////////////////////////////////////////// - -`define NW_BITS `CLOG2(`NUM_WARPS) -`define NC_WIDTH `UP(`NC_BITS) - -`define NT_BITS `CLOG2(`NUM_THREADS) -`define NW_WIDTH `UP(`NW_BITS) - -`define NC_BITS `CLOG2(`NUM_CORES) -`define NT_WIDTH `UP(`NT_BITS) - -`define NB_BITS `CLOG2(`NUM_BARRIERS) -`define NB_WIDTH `UP(`NB_BITS) - -`define NUM_IREGS 32 - -`define NRI_BITS `CLOG2(`NUM_IREGS) - -`ifdef EXT_F_ENABLE -`define NUM_REGS (2 * `NUM_IREGS) -`else -`define NUM_REGS `NUM_IREGS -`endif - -`define NR_BITS `CLOG2(`NUM_REGS) - -`define PERF_CTR_BITS 44 - -`ifndef NDEBUG -`define UUID_WIDTH 44 -`else -`define UUID_WIDTH 1 -`endif - -/////////////////////////////////////////////////////////////////////////////// - -`define EX_ALU 0 -`define EX_LSU 1 -`define EX_SFU 2 -`define EX_FPU 3 - -`define NUM_EX_UNITS (3 + `EXT_F_ENABLED) -`define EX_BITS `CLOG2(`NUM_EX_UNITS) - -/////////////////////////////////////////////////////////////////////////////// - -`define INST_LUI 7'b0110111 -`define INST_AUIPC 7'b0010111 -`define INST_JAL 7'b1101111 -`define INST_JALR 7'b1100111 -`define INST_B 7'b1100011 // branch instructions -`define INST_L 7'b0000011 // load instructions -`define INST_S 7'b0100011 // store instructions -`define INST_I 7'b0010011 // immediate instructions -`define INST_R 7'b0110011 // register instructions -`define INST_FENCE 7'b0001111 // Fence instructions -`define INST_SYS 7'b1110011 // system instructions - -// RV64I instruction specific opcodes (for any W instruction) -`define INST_I_W 7'b0011011 // W type immediate instructions -`define INST_R_W 7'b0111011 // W type register instructions - -`define INST_FL 7'b0000111 // float load instruction -`define INST_FS 7'b0100111 // float store instruction -`define INST_FMADD 7'b1000011 -`define INST_FMSUB 7'b1000111 -`define INST_FNMSUB 7'b1001011 -`define INST_FNMADD 7'b1001111 -`define INST_FCI 7'b1010011 // float common instructions - -// Custom extension opcodes -`define INST_EXT1 7'b0001011 // 0x0B -`define INST_EXT2 7'b0101011 // 0x2B -`define INST_EXT3 7'b1011011 // 0x5B -`define INST_EXT4 7'b1111011 // 0x7B - -/////////////////////////////////////////////////////////////////////////////// - -`define INST_FRM_RNE 3'b000 // round to nearest even -`define INST_FRM_RTZ 3'b001 // round to zero -`define INST_FRM_RDN 3'b010 // round to -inf -`define INST_FRM_RUP 3'b011 // round to +inf -`define INST_FRM_RMM 3'b100 // round to nearest max magnitude -`define INST_FRM_DYN 3'b111 // dynamic mode -`define INST_FRM_BITS 3 - -/////////////////////////////////////////////////////////////////////////////// - -`define INST_OP_BITS 4 -`define INST_MOD_BITS 3 -`define INST_FMT_BITS 2 - -/////////////////////////////////////////////////////////////////////////////// - -`define INST_ALU_ADD 4'b0000 -`define INST_ALU_LUI 4'b0010 -`define INST_ALU_AUIPC 4'b0011 -`define INST_ALU_SLTU 4'b0100 -`define INST_ALU_SLT 4'b0101 -`define INST_ALU_SUB 4'b0111 -`define INST_ALU_SRL 4'b1000 -`define INST_ALU_SRA 4'b1001 -`define INST_ALU_AND 4'b1100 -`define INST_ALU_OR 4'b1101 -`define INST_ALU_XOR 4'b1110 -`define INST_ALU_SLL 4'b1111 -`define INST_ALU_OTHER 4'b0111 -`define INST_ALU_BITS 4 -`define INST_ALU_CLASS(op) op[3:2] -`define INST_ALU_SIGNED(op) op[0] -`define INST_ALU_IS_SUB(op) op[1] -`define INST_ALU_IS_BR(mod) mod[0] -`define INST_ALU_IS_M(mod) mod[1] -`define INST_ALU_IS_W(mod) mod[2] - -`define INST_BR_EQ 4'b0000 -`define INST_BR_NE 4'b0010 -`define INST_BR_LTU 4'b0100 -`define INST_BR_GEU 4'b0110 -`define INST_BR_LT 4'b0101 -`define INST_BR_GE 4'b0111 -`define INST_BR_JAL 4'b1000 -`define INST_BR_JALR 4'b1001 -`define INST_BR_ECALL 4'b1010 -`define INST_BR_EBREAK 4'b1011 -`define INST_BR_URET 4'b1100 -`define INST_BR_SRET 4'b1101 -`define INST_BR_MRET 4'b1110 -`define INST_BR_OTHER 4'b1111 -`define INST_BR_BITS 4 -`define INST_BR_CLASS(op) {1'b0, ~op[3]} -`define INST_BR_IS_NEG(op) op[1] -`define INST_BR_IS_LESS(op) op[2] -`define INST_BR_IS_STATIC(op) op[3] - -`define INST_M_MUL 3'b000 -`define INST_M_MULHU 3'b001 -`define INST_M_MULH 3'b010 -`define INST_M_MULHSU 3'b011 -`define INST_M_DIV 3'b100 -`define INST_M_DIVU 3'b101 -`define INST_M_REM 3'b110 -`define INST_M_REMU 3'b111 -`define INST_M_BITS 3 -`define INST_M_SIGNED(op) (~op[0]) -`define INST_M_IS_MULX(op) (~op[2]) -`define INST_M_IS_MULH(op) (op[1:0] != 0) -`define INST_M_SIGNED_A(op) (op[1:0] != 1) -`define INST_M_IS_REM(op) op[1] - -`define INST_FMT_B 3'b000 -`define INST_FMT_H 3'b001 -`define INST_FMT_W 3'b010 -`define INST_FMT_D 3'b011 -`define INST_FMT_BU 3'b100 -`define INST_FMT_HU 3'b101 -`define INST_FMT_WU 3'b110 - -`define INST_LSU_LB 4'b0000 -`define INST_LSU_LH 4'b0001 -`define INST_LSU_LW 4'b0010 -`define INST_LSU_LD 4'b0011 // new for RV64I LD -`define INST_LSU_LBU 4'b0100 -`define INST_LSU_LHU 4'b0101 -`define INST_LSU_LWU 4'b0110 // new for RV64I LWU -`define INST_LSU_SB 4'b1000 -`define INST_LSU_SH 4'b1001 -`define INST_LSU_SW 4'b1010 -`define INST_LSU_SD 4'b1011 // new for RV64I SD -`define INST_LSU_FENCE 4'b1111 -`define INST_LSU_BITS 4 -`define INST_LSU_FMT(op) op[2:0] -`define INST_LSU_WSIZE(op) op[1:0] -`define INST_LSU_IS_FENCE(op) (op[3:2] == 3) - -`define INST_FENCE_BITS 1 -`define INST_FENCE_D 1'h0 -`define INST_FENCE_I 1'h1 - -`define INST_FPU_ADD 4'b0000 -`define INST_FPU_SUB 4'b0001 -`define INST_FPU_MUL 4'b0010 -`define INST_FPU_DIV 4'b0011 -`define INST_FPU_SQRT 4'b0100 -`define INST_FPU_CMP 4'b0101 // mod: LE=0, LT=1, EQ=2 -`define INST_FPU_F2F 4'b0110 -`define INST_FPU_MISC 4'b0111 // mod: SGNJ=0, SGNJN=1, SGNJX=2, CLASS=3, MVXW=4, MVWX=5, FMIN=6, FMAX=7 -`define INST_FPU_F2I 4'b1000 -`define INST_FPU_F2U 4'b1001 -`define INST_FPU_I2F 4'b1010 -`define INST_FPU_U2F 4'b1011 -`define INST_FPU_MADD 4'b1100 -`define INST_FPU_MSUB 4'b1101 -`define INST_FPU_NMSUB 4'b1110 -`define INST_FPU_NMADD 4'b1111 -`define INST_FPU_BITS 4 -`define INST_FPU_IS_W(mod) (mod[4]) -`define INST_FPU_IS_CLASS(op, mod) (op == `INST_FPU_MISC && mod == 3) -`define INST_FPU_IS_MVXW(op, mod) (op == `INST_FPU_MISC && mod == 4) - -`define INST_SFU_TMC 4'h0 -`define INST_SFU_WSPAWN 4'h1 -`define INST_SFU_SPLIT 4'h2 -`define INST_SFU_JOIN 4'h3 -`define INST_SFU_BAR 4'h4 -`define INST_SFU_PRED 4'h5 -`define INST_SFU_CSRRW 4'h6 -`define INST_SFU_CSRRS 4'h7 -`define INST_SFU_CSRRC 4'h8 -`define INST_SFU_CMOV 4'h9 -`define INST_SFU_BITS 4 -`define INST_SFU_CSR(f3) (4'h6 + 4'(f3) - 4'h1) -`define INST_SFU_IS_WCTL(op) (op <= 5) -`define INST_SFU_IS_CSR(op) (op >= 6 && op <= 8) - -/////////////////////////////////////////////////////////////////////////////// - -// non-cacheable tag bits -`define NC_TAG_BITS 1 - -// cache address type bits -`ifdef SM_ENABLE -`define CACHE_ADDR_TYPE_BITS (`NC_TAG_BITS + 1) -`else -`define CACHE_ADDR_TYPE_BITS `NC_TAG_BITS -`endif - -`define ARB_SEL_BITS(I, O) ((I > O) ? `CLOG2((I + O - 1) / O) : 0) - -/////////////////////////////////////////////////////////////////////////////// - -`define CACHE_MEM_TAG_WIDTH(mshr_size, num_banks) \ - (`CLOG2(mshr_size) + `CLOG2(num_banks) + `NC_TAG_BITS) - -`define CACHE_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width) \ - (`CLOG2(num_reqs) + `CLOG2(line_size / word_size) + tag_width) - -`define CACHE_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width) \ - (`CACHE_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width) + `NC_TAG_BITS) - -`define CACHE_NC_MEM_TAG_WIDTH(mshr_size, num_banks, num_reqs, line_size, word_size, tag_width) \ - `MAX(`CACHE_MEM_TAG_WIDTH(mshr_size, num_banks), `CACHE_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width)) - -/////////////////////////////////////////////////////////////////////////////// - -`define CACHE_CLUSTER_CORE_ARB_TAG(tag_width, num_inputs, num_caches) \ - (tag_width + `ARB_SEL_BITS(num_inputs, `UP(num_caches))) - -`define CACHE_CLUSTER_MEM_ARB_TAG(tag_width, num_caches) \ - (tag_width + `ARB_SEL_BITS(`UP(num_caches), 1)) - -`define CACHE_CLUSTER_MEM_TAG_WIDTH(mshr_size, num_banks, num_caches) \ - `CACHE_CLUSTER_MEM_ARB_TAG(`CACHE_MEM_TAG_WIDTH(mshr_size, num_banks), num_caches) - -`define CACHE_CLUSTER_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width, num_inputs, num_caches) \ - `CACHE_CLUSTER_MEM_ARB_TAG((`CLOG2(num_reqs) + `CLOG2(line_size / word_size) + `CACHE_CLUSTER_CORE_ARB_TAG(tag_width, num_inputs, num_caches)), num_caches) - -`define CACHE_CLUSTER_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width, num_inputs, num_caches) \ - `CACHE_CLUSTER_MEM_ARB_TAG((`CACHE_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, `CACHE_CLUSTER_CORE_ARB_TAG(tag_width, num_inputs, num_caches)) + `NC_TAG_BITS), num_caches) - -`define CACHE_CLUSTER_NC_MEM_TAG_WIDTH(mshr_size, num_banks, num_reqs, line_size, word_size, tag_width, num_inputs, num_caches) \ - `CACHE_CLUSTER_MEM_ARB_TAG(`MAX(`CACHE_MEM_TAG_WIDTH(mshr_size, num_banks), `CACHE_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, `CACHE_CLUSTER_CORE_ARB_TAG(tag_width, num_inputs, num_caches))), num_caches) - -/////////////////////////////////////////////////////////////////////////////// - -`ifdef L2_ENABLE -`define L2_LINE_SIZE `MEM_BLOCK_SIZE -`else -`define L2_LINE_SIZE `L1_LINE_SIZE -`endif - -`ifdef L3_ENABLE -`define L3_LINE_SIZE `MEM_BLOCK_SIZE -`else -`define L3_LINE_SIZE `L2_LINE_SIZE -`endif - -`define VX_MEM_BYTEEN_WIDTH `L3_LINE_SIZE -`define VX_MEM_ADDR_WIDTH (`MEM_ADDR_WIDTH - `CLOG2(`L3_LINE_SIZE)) -`define VX_MEM_DATA_WIDTH (`L3_LINE_SIZE * 8) -`define VX_MEM_TAG_WIDTH L3_MEM_TAG_WIDTH - -`define VX_DCR_ADDR_WIDTH `VX_DCR_ADDR_BITS -`define VX_DCR_DATA_WIDTH 32 - -`define TO_FULL_ADDR(x) {x, (`MEM_ADDR_WIDTH-$bits(x))'(0)} - -/////////////////////////////////////////////////////////////////////////////// - -`define BUFFER_BUSY(dst, src, enable) \ - logic __busy; \ - if (enable) begin \ - always @(posedge clk) begin \ - if (reset) begin \ - __busy <= 1'b0; \ - end else begin \ - __busy <= src; \ - end \ - end \ - end else begin \ - assign __busy = src; \ - end \ - assign dst = __busy - -`define POP_COUNT_EX(out, in, model) \ - VX_popcount #( \ - .N ($bits(in)), \ - .MODEL (model) \ - ) __``out ( \ - .data_in (in), \ - .data_out (out) \ - ) - -`define POP_COUNT(out, in) `POP_COUNT_EX(out, in, 1) - -`define ASSIGN_VX_MEM_BUS_IF(dst, src) \ - assign dst.req_valid = src.req_valid; \ - assign dst.req_data = src.req_data; \ - assign src.req_ready = dst.req_ready; \ - assign src.rsp_valid = dst.rsp_valid; \ - assign src.rsp_data = dst.rsp_data; \ - assign dst.rsp_ready = src.rsp_ready - -`define ASSIGN_VX_MEM_BUS_IF_X(dst, src, TD, TS) \ - assign dst.req_valid = src.req_valid; \ - assign dst.req_data.rw = src.req_data.rw; \ - assign dst.req_data.byteen = src.req_data.byteen; \ - assign dst.req_data.addr = src.req_data.addr; \ - assign dst.req_data.data = src.req_data.data; \ - if (TD != TS) \ - assign dst.req_data.tag = {src.req_data.tag, {(TD-TS){1'b0}}}; \ - else \ - assign dst.req_data.tag = src.req_data.tag; \ - assign src.req_ready = dst.req_ready; \ - assign src.rsp_valid = dst.rsp_valid; \ - assign src.rsp_data.data = dst.rsp_data.data; \ - assign src.rsp_data.tag = dst.rsp_data.tag[TD-1 -: TS]; \ - assign dst.rsp_ready = src.rsp_ready - -`define BUFFER_DCR_BUS_IF(dst, src, enable) \ - logic [(1 + `VX_DCR_ADDR_WIDTH + `VX_DCR_DATA_WIDTH)-1:0] __``dst; \ - if (enable) begin \ - always @(posedge clk) begin \ - __``dst <= {src.write_valid, src.write_addr, src.write_data}; \ - end \ - end else begin \ - assign __``dst = {src.write_valid, src.write_addr, src.write_data}; \ - end \ - VX_dcr_bus_if dst(); \ - assign {dst.write_valid, dst.write_addr, dst.write_data} = __``dst - -`define PERF_REDUCE(dst, src, field, width, count) \ - wire [count-1:0][width-1:0] __reduce_add_i_``src``field; \ - wire [width-1:0] __reduce_add_o_``dst``field; \ - reg [width-1:0] __reduce_add_r_``dst``field; \ - for (genvar __i = 0; __i < count; ++__i) begin \ - assign __reduce_add_i_``src``field[__i] = ``src[__i].``field; \ - end \ - VX_reduce #(.DATAW_IN(width), .N(count), .OP("+")) __reduce_add_``dst``field ( \ - __reduce_add_i_``src``field, \ - __reduce_add_o_``dst``field \ - ); \ - always @(posedge clk) begin \ - if (reset) begin \ - __reduce_add_r_``dst``field <= '0; \ - end else begin \ - __reduce_add_r_``dst``field <= __reduce_add_o_``dst``field; \ - end \ - end \ - assign ``dst.``field = __reduce_add_r_``dst``field - -`define PERF_CACHE_REDUCE(dst, src, count) \ - `PERF_REDUCE (dst, src, reads, `PERF_CTR_BITS, count); \ - `PERF_REDUCE (dst, src, writes, `PERF_CTR_BITS, count); \ - `PERF_REDUCE (dst, src, read_misses, `PERF_CTR_BITS, count); \ - `PERF_REDUCE (dst, src, write_misses, `PERF_CTR_BITS, count); \ - `PERF_REDUCE (dst, src, bank_stalls, `PERF_CTR_BITS, count); \ - `PERF_REDUCE (dst, src, mshr_stalls, `PERF_CTR_BITS, count); \ - `PERF_REDUCE (dst, src, mem_stalls, `PERF_CTR_BITS, count); \ - `PERF_REDUCE (dst, src, crsp_stalls, `PERF_CTR_BITS, count) - -`define ASSIGN_BLOCKED_WID(dst, src, block_idx, block_size) \ - if (block_size != 1) begin \ - if (block_size != `NUM_WARPS) begin \ - assign dst = {src[`NW_WIDTH-1:`CLOG2(block_size)], `CLOG2(block_size)'(block_idx)}; \ - end else begin \ - assign dst = `NW_WIDTH'(block_idx); \ - end \ - end else begin \ - assign dst = src; \ - end - -`define TO_DISPATCH_DATA(data, tid) \ - {data.uuid, data.wis, data.tmask, data.op_type, data.op_mod, data.wb, data.use_PC, data.use_imm, data.PC, data.imm, data.rd, tid, data.rs1_data, data.rs2_data, data.rs3_data} - -/////////////////////////////////////////////////////////////////////////////// - -`endif // VX_DEFINE_VH +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`ifndef VX_DEFINE_VH +`define VX_DEFINE_VH + +`include "VX_platform.vh" +`include "VX_config.vh" +`include "VX_types.vh" + +/////////////////////////////////////////////////////////////////////////////// + +`define NW_BITS `CLOG2(`NUM_WARPS) +`define NC_WIDTH `UP(`NC_BITS) + +`define NT_BITS `CLOG2(`NUM_THREADS) +`define NW_WIDTH `UP(`NW_BITS) + +`define NC_BITS `CLOG2(`NUM_CORES) +`define NT_WIDTH `UP(`NT_BITS) + +`define NB_BITS `CLOG2(`NUM_BARRIERS) +`define NB_WIDTH `UP(`NB_BITS) + +`define NUM_IREGS 32 + +`define NRI_BITS `CLOG2(`NUM_IREGS) + +`ifdef EXT_F_ENABLE +`define NUM_REGS (2 * `NUM_IREGS) +`else +`define NUM_REGS `NUM_IREGS +`endif + +`define NR_BITS `CLOG2(`NUM_REGS) + +`define PERF_CTR_BITS 44 + +`ifndef NDEBUG +`define UUID_WIDTH 44 +`else +`define UUID_WIDTH 1 +`endif + +/////////////////////////////////////////////////////////////////////////////// + +`define EX_ALU 0 +`define EX_LSU 1 +`define EX_SFU 2 +`define EX_FPU (`EX_SFU + `EXT_F_ENABLED) + +`define NUM_EX_UNITS (3 + `EXT_F_ENABLED) +`define EX_BITS `CLOG2(`NUM_EX_UNITS) +`define EX_WIDTH `UP(`EX_BITS) + +`define SFU_CSRS 0 +`define SFU_WCTL 1 + +`define NUM_SFU_UNITS (2) +`define SFU_BITS `CLOG2(`NUM_SFU_UNITS) +`define SFU_WIDTH `UP(`SFU_BITS) + +/////////////////////////////////////////////////////////////////////////////// + +`define INST_LUI 7'b0110111 +`define INST_AUIPC 7'b0010111 +`define INST_JAL 7'b1101111 +`define INST_JALR 7'b1100111 +`define INST_B 7'b1100011 // branch instructions +`define INST_L 7'b0000011 // load instructions +`define INST_S 7'b0100011 // store instructions +`define INST_I 7'b0010011 // immediate instructions +`define INST_R 7'b0110011 // register instructions +`define INST_FENCE 7'b0001111 // Fence instructions +`define INST_SYS 7'b1110011 // system instructions + +// RV64I instruction specific opcodes (for any W instruction) +`define INST_I_W 7'b0011011 // W type immediate instructions +`define INST_R_W 7'b0111011 // W type register instructions + +`define INST_FL 7'b0000111 // float load instruction +`define INST_FS 7'b0100111 // float store instruction +`define INST_FMADD 7'b1000011 +`define INST_FMSUB 7'b1000111 +`define INST_FNMSUB 7'b1001011 +`define INST_FNMADD 7'b1001111 +`define INST_FCI 7'b1010011 // float common instructions + +// Custom extension opcodes +`define INST_EXT1 7'b0001011 // 0x0B +`define INST_EXT2 7'b0101011 // 0x2B +`define INST_EXT3 7'b1011011 // 0x5B +`define INST_EXT4 7'b1111011 // 0x7B + +/////////////////////////////////////////////////////////////////////////////// + +`define INST_FRM_RNE 3'b000 // round to nearest even +`define INST_FRM_RTZ 3'b001 // round to zero +`define INST_FRM_RDN 3'b010 // round to -inf +`define INST_FRM_RUP 3'b011 // round to +inf +`define INST_FRM_RMM 3'b100 // round to nearest max magnitude +`define INST_FRM_DYN 3'b111 // dynamic mode +`define INST_FRM_BITS 3 + +/////////////////////////////////////////////////////////////////////////////// + +`define INST_OP_BITS 4 +`define INST_MOD_BITS 3 +`define INST_FMT_BITS 2 + +/////////////////////////////////////////////////////////////////////////////// + +`define INST_ALU_ADD 4'b0000 +`define INST_ALU_LUI 4'b0010 +`define INST_ALU_AUIPC 4'b0011 +`define INST_ALU_SLTU 4'b0100 +`define INST_ALU_SLT 4'b0101 +`define INST_ALU_SUB 4'b0111 +`define INST_ALU_SRL 4'b1000 +`define INST_ALU_SRA 4'b1001 +`define INST_ALU_AND 4'b1100 +`define INST_ALU_OR 4'b1101 +`define INST_ALU_XOR 4'b1110 +`define INST_ALU_SLL 4'b1111 +`define INST_ALU_OTHER 4'b0111 +`define INST_ALU_BITS 4 +`define INST_ALU_CLASS(op) op[3:2] +`define INST_ALU_SIGNED(op) op[0] +`define INST_ALU_IS_SUB(op) op[1] +`define INST_ALU_IS_BR(mod) mod[0] +`define INST_ALU_IS_M(mod) mod[1] +`define INST_ALU_IS_W(mod) mod[2] + +`define INST_BR_EQ 4'b0000 +`define INST_BR_NE 4'b0010 +`define INST_BR_LTU 4'b0100 +`define INST_BR_GEU 4'b0110 +`define INST_BR_LT 4'b0101 +`define INST_BR_GE 4'b0111 +`define INST_BR_JAL 4'b1000 +`define INST_BR_JALR 4'b1001 +`define INST_BR_ECALL 4'b1010 +`define INST_BR_EBREAK 4'b1011 +`define INST_BR_URET 4'b1100 +`define INST_BR_SRET 4'b1101 +`define INST_BR_MRET 4'b1110 +`define INST_BR_OTHER 4'b1111 +`define INST_BR_BITS 4 +`define INST_BR_CLASS(op) {1'b0, ~op[3]} +`define INST_BR_IS_NEG(op) op[1] +`define INST_BR_IS_LESS(op) op[2] +`define INST_BR_IS_STATIC(op) op[3] + +`define INST_M_MUL 3'b000 +`define INST_M_MULHU 3'b001 +`define INST_M_MULH 3'b010 +`define INST_M_MULHSU 3'b011 +`define INST_M_DIV 3'b100 +`define INST_M_DIVU 3'b101 +`define INST_M_REM 3'b110 +`define INST_M_REMU 3'b111 +`define INST_M_BITS 3 +`define INST_M_SIGNED(op) (~op[0]) +`define INST_M_IS_MULX(op) (~op[2]) +`define INST_M_IS_MULH(op) (op[1:0] != 0) +`define INST_M_SIGNED_A(op) (op[1:0] != 1) +`define INST_M_IS_REM(op) op[1] + +`define INST_FMT_B 3'b000 +`define INST_FMT_H 3'b001 +`define INST_FMT_W 3'b010 +`define INST_FMT_D 3'b011 +`define INST_FMT_BU 3'b100 +`define INST_FMT_HU 3'b101 +`define INST_FMT_WU 3'b110 + +`define INST_LSU_LB 4'b0000 +`define INST_LSU_LH 4'b0001 +`define INST_LSU_LW 4'b0010 +`define INST_LSU_LD 4'b0011 // new for RV64I LD +`define INST_LSU_LBU 4'b0100 +`define INST_LSU_LHU 4'b0101 +`define INST_LSU_LWU 4'b0110 // new for RV64I LWU +`define INST_LSU_SB 4'b1000 +`define INST_LSU_SH 4'b1001 +`define INST_LSU_SW 4'b1010 +`define INST_LSU_SD 4'b1011 // new for RV64I SD +`define INST_LSU_FENCE 4'b1111 +`define INST_LSU_BITS 4 +`define INST_LSU_FMT(op) op[2:0] +`define INST_LSU_WSIZE(op) op[1:0] +`define INST_LSU_IS_FENCE(op) (op[3:2] == 3) + +`define INST_FENCE_BITS 1 +`define INST_FENCE_D 1'h0 +`define INST_FENCE_I 1'h1 + +`define INST_FPU_ADD 4'b0000 +`define INST_FPU_SUB 4'b0001 +`define INST_FPU_MUL 4'b0010 +`define INST_FPU_DIV 4'b0011 +`define INST_FPU_SQRT 4'b0100 +`define INST_FPU_CMP 4'b0101 // mod: LE=0, LT=1, EQ=2 +`define INST_FPU_F2F 4'b0110 +`define INST_FPU_MISC 4'b0111 // mod: SGNJ=0, SGNJN=1, SGNJX=2, CLASS=3, MVXW=4, MVWX=5, FMIN=6, FMAX=7 +`define INST_FPU_F2I 4'b1000 +`define INST_FPU_F2U 4'b1001 +`define INST_FPU_I2F 4'b1010 +`define INST_FPU_U2F 4'b1011 +`define INST_FPU_MADD 4'b1100 +`define INST_FPU_MSUB 4'b1101 +`define INST_FPU_NMSUB 4'b1110 +`define INST_FPU_NMADD 4'b1111 +`define INST_FPU_BITS 4 +`define INST_FPU_IS_W(mod) (mod[4]) +`define INST_FPU_IS_CLASS(op, mod) (op == `INST_FPU_MISC && mod == 3) +`define INST_FPU_IS_MVXW(op, mod) (op == `INST_FPU_MISC && mod == 4) + +`define INST_SFU_TMC 4'h0 +`define INST_SFU_WSPAWN 4'h1 +`define INST_SFU_SPLIT 4'h2 +`define INST_SFU_JOIN 4'h3 +`define INST_SFU_BAR 4'h4 +`define INST_SFU_PRED 4'h5 +`define INST_SFU_CSRRW 4'h6 +`define INST_SFU_CSRRS 4'h7 +`define INST_SFU_CSRRC 4'h8 +`define INST_SFU_CMOV 4'h9 +`define INST_SFU_BITS 4 +`define INST_SFU_CSR(f3) (4'h6 + 4'(f3) - 4'h1) +`define INST_SFU_IS_WCTL(op) (op <= 5) +`define INST_SFU_IS_CSR(op) (op >= 6 && op <= 8) + +/////////////////////////////////////////////////////////////////////////////// + +// non-cacheable tag bits +`define NC_TAG_BITS 1 + +// cache address type bits +`ifdef SM_ENABLE +`define CACHE_ADDR_TYPE_BITS (`NC_TAG_BITS + 1) +`else +`define CACHE_ADDR_TYPE_BITS `NC_TAG_BITS +`endif + +`define ARB_SEL_BITS(I, O) ((I > O) ? `CLOG2((I + O - 1) / O) : 0) + +/////////////////////////////////////////////////////////////////////////////// + +`define CACHE_MEM_TAG_WIDTH(mshr_size, num_banks) \ + (`CLOG2(mshr_size) + `CLOG2(num_banks) + `NC_TAG_BITS) + +`define CACHE_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width) \ + (`CLOG2(num_reqs) + `CLOG2(line_size / word_size) + tag_width) + +`define CACHE_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width) \ + (`CACHE_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width) + `NC_TAG_BITS) + +`define CACHE_NC_MEM_TAG_WIDTH(mshr_size, num_banks, num_reqs, line_size, word_size, tag_width) \ + `MAX(`CACHE_MEM_TAG_WIDTH(mshr_size, num_banks), `CACHE_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width)) + +/////////////////////////////////////////////////////////////////////////////// + +`define CACHE_CLUSTER_CORE_ARB_TAG(tag_width, num_inputs, num_caches) \ + (tag_width + `ARB_SEL_BITS(num_inputs, `UP(num_caches))) + +`define CACHE_CLUSTER_MEM_ARB_TAG(tag_width, num_caches) \ + (tag_width + `ARB_SEL_BITS(`UP(num_caches), 1)) + +`define CACHE_CLUSTER_MEM_TAG_WIDTH(mshr_size, num_banks, num_caches) \ + `CACHE_CLUSTER_MEM_ARB_TAG(`CACHE_MEM_TAG_WIDTH(mshr_size, num_banks), num_caches) + +`define CACHE_CLUSTER_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width, num_inputs, num_caches) \ + `CACHE_CLUSTER_MEM_ARB_TAG((`CLOG2(num_reqs) + `CLOG2(line_size / word_size) + `CACHE_CLUSTER_CORE_ARB_TAG(tag_width, num_inputs, num_caches)), num_caches) + +`define CACHE_CLUSTER_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, tag_width, num_inputs, num_caches) \ + `CACHE_CLUSTER_MEM_ARB_TAG((`CACHE_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, `CACHE_CLUSTER_CORE_ARB_TAG(tag_width, num_inputs, num_caches)) + `NC_TAG_BITS), num_caches) + +`define CACHE_CLUSTER_NC_MEM_TAG_WIDTH(mshr_size, num_banks, num_reqs, line_size, word_size, tag_width, num_inputs, num_caches) \ + `CACHE_CLUSTER_MEM_ARB_TAG(`MAX(`CACHE_MEM_TAG_WIDTH(mshr_size, num_banks), `CACHE_NC_BYPASS_TAG_WIDTH(num_reqs, line_size, word_size, `CACHE_CLUSTER_CORE_ARB_TAG(tag_width, num_inputs, num_caches))), num_caches) + +/////////////////////////////////////////////////////////////////////////////// + +`ifdef L2_ENABLE +`define L2_LINE_SIZE `MEM_BLOCK_SIZE +`else +`define L2_LINE_SIZE `L1_LINE_SIZE +`endif + +`ifdef L3_ENABLE +`define L3_LINE_SIZE `MEM_BLOCK_SIZE +`else +`define L3_LINE_SIZE `L2_LINE_SIZE +`endif + +`define VX_MEM_BYTEEN_WIDTH `L3_LINE_SIZE +`define VX_MEM_ADDR_WIDTH (`MEM_ADDR_WIDTH - `CLOG2(`L3_LINE_SIZE)) +`define VX_MEM_DATA_WIDTH (`L3_LINE_SIZE * 8) +`define VX_MEM_TAG_WIDTH L3_MEM_TAG_WIDTH + +`define VX_DCR_ADDR_WIDTH `VX_DCR_ADDR_BITS +`define VX_DCR_DATA_WIDTH 32 + +`define TO_FULL_ADDR(x) {x, (`MEM_ADDR_WIDTH-$bits(x))'(0)} + +/////////////////////////////////////////////////////////////////////////////// + +`define BUFFER_EX(dst, src, ena, latency) \ + VX_pipe_register #( \ + .DATAW ($bits(dst)), \ + .RESETW ($bits(dst)), \ + .DEPTH (latency) \ + ) __``dst ( \ + .clk (clk), \ + .reset (reset), \ + .enable (ena), \ + .data_in (src), \ + .data_out (dst) \ + ) + +`define BUFFER(dst, src) `BUFFER_EX(dst, src, 1'b1, 1) + +`define POP_COUNT_EX(out, in, model) \ + VX_popcount #( \ + .N ($bits(in)), \ + .MODEL (model) \ + ) __``out ( \ + .data_in (in), \ + .data_out (out) \ + ) + +`define POP_COUNT(out, in) `POP_COUNT_EX(out, in, 1) + +`define ASSIGN_VX_MEM_BUS_IF(dst, src) \ + assign dst.req_valid = src.req_valid; \ + assign dst.req_data = src.req_data; \ + assign src.req_ready = dst.req_ready; \ + assign src.rsp_valid = dst.rsp_valid; \ + assign src.rsp_data = dst.rsp_data; \ + assign dst.rsp_ready = src.rsp_ready + +`define ASSIGN_VX_MEM_BUS_IF_X(dst, src, TD, TS) \ + assign dst.req_valid = src.req_valid; \ + assign dst.req_data.rw = src.req_data.rw; \ + assign dst.req_data.byteen = src.req_data.byteen; \ + assign dst.req_data.addr = src.req_data.addr; \ + assign dst.req_data.data = src.req_data.data; \ + if (TD != TS) \ + assign dst.req_data.tag = {src.req_data.tag, {(TD-TS){1'b0}}}; \ + else \ + assign dst.req_data.tag = src.req_data.tag; \ + assign src.req_ready = dst.req_ready; \ + assign src.rsp_valid = dst.rsp_valid; \ + assign src.rsp_data.data = dst.rsp_data.data; \ + assign src.rsp_data.tag = dst.rsp_data.tag[TD-1 -: TS]; \ + assign dst.rsp_ready = src.rsp_ready + +`define BUFFER_DCR_BUS_IF(dst, src, enable) \ + logic [(1 + `VX_DCR_ADDR_WIDTH + `VX_DCR_DATA_WIDTH)-1:0] __``dst; \ + if (enable) begin \ + always @(posedge clk) begin \ + __``dst <= {src.write_valid, src.write_addr, src.write_data}; \ + end \ + end else begin \ + assign __``dst = {src.write_valid, src.write_addr, src.write_data}; \ + end \ + VX_dcr_bus_if dst(); \ + assign {dst.write_valid, dst.write_addr, dst.write_data} = __``dst + +`define PERF_COUNTER_ADD(dst, src, field, width, dst_count, src_count, reg_enable) \ + for (genvar __d = 0; __d < dst_count; ++__d) begin \ + localparam __count = ((src_count > dst_count) ? ((src_count + dst_count - 1) / dst_count) : 1); \ + wire [__count-1:0][width-1:0] __reduce_add_i_``src``field; \ + wire [width-1:0] __reduce_add_o_``dst``field; \ + for (genvar __i = 0; __i < __count; ++__i) begin \ + assign __reduce_add_i_``src``field[__i] = ``src[__d * __count + __i].``field; \ + end \ + VX_reduce #(.DATAW_IN(width), .N(__count), .OP("+")) __reduce_add_``dst``field ( \ + __reduce_add_i_``src``field, \ + __reduce_add_o_``dst``field \ + ); \ + if (reg_enable) begin \ + reg [width-1:0] __reduce_add_r_``dst``field; \ + always @(posedge clk) begin \ + if (reset) begin \ + __reduce_add_r_``dst``field <= '0; \ + end else begin \ + __reduce_add_r_``dst``field <= __reduce_add_o_``dst``field; \ + end \ + end \ + assign ``dst[__d].``field = __reduce_add_r_``dst``field; \ + end else begin \ + assign ``dst[__d].``field = __reduce_add_o_``dst``field; \ + end \ + end + +`define ASSIGN_BLOCKED_WID(dst, src, block_idx, block_size) \ + if (block_size != 1) begin \ + if (block_size != `NUM_WARPS) begin \ + assign dst = {src[`NW_WIDTH-1:`CLOG2(block_size)], `CLOG2(block_size)'(block_idx)}; \ + end else begin \ + assign dst = `NW_WIDTH'(block_idx); \ + end \ + end else begin \ + assign dst = src; \ + end + +`define TO_DISPATCH_DATA(data, tid) { \ + data.uuid, \ + data.wis, \ + data.tmask, \ + data.op_type, \ + data.op_mod, \ + data.wb, \ + data.use_PC, \ + data.use_imm, \ + data.PC, \ + data.imm, \ + data.rd, \ + tid, \ + data.rs1_data, \ + data.rs2_data, \ + data.rs3_data} + +/////////////////////////////////////////////////////////////////////////////// + +`endif // VX_DEFINE_VH diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index cdb48db4..49dc9564 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -99,7 +99,7 @@ package VX_gpu_pkg; `ifdef ICACHE_ENABLE localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_MEM_TAG_WIDTH(`ICACHE_MSHR_SIZE, 1, `NUM_ICACHES); `else - localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_BYPASS_TAG_WIDTH(1, ICACHE_LINE_SIZE, ICACHE_WORD_SIZE, ICACHE_TAG_WIDTH, `NUM_SOCKETS, `NUM_ICACHES); + localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_BYPASS_TAG_WIDTH(1, ICACHE_LINE_SIZE, ICACHE_WORD_SIZE, ICACHE_TAG_WIDTH, `SOCKET_SIZE, `NUM_ICACHES); `endif ////////////////////////// Dcache Parameters ////////////////////////////// @@ -142,10 +142,13 @@ package VX_gpu_pkg; /////////////////////////////// L1 Parameters ///////////////////////////// localparam L1_MEM_TAG_WIDTH = `MAX(ICACHE_MEM_TAG_WIDTH, DCACHE_MEM_TAG_WIDTH); - localparam L1_MEM_ARB_TAG_WIDTH = (L1_MEM_TAG_WIDTH + `CLOG2(2)); - + localparam L1_MEM_ARB_TAG_WIDTH = (L1_MEM_TAG_WIDTH + `CLOG2(2)); + /////////////////////////////// L2 Parameters ///////////////////////////// + localparam ICACHE_MEM_ARB_IDX = 0; + localparam DCACHE_MEM_ARB_IDX = ICACHE_MEM_ARB_IDX + 1; + // Word size in bytes localparam L2_WORD_SIZE = `L1_LINE_SIZE; @@ -190,42 +193,46 @@ package VX_gpu_pkg; /////////////////////////////// Issue parameters ////////////////////////// - localparam ISSUE_IDX_W = `LOG2UP(`ISSUE_WIDTH); + localparam ISSUE_ISW = `CLOG2(`ISSUE_WIDTH); + localparam ISSUE_ISW_W = `UP(ISSUE_ISW); localparam ISSUE_RATIO = `NUM_WARPS / `ISSUE_WIDTH; - localparam ISSUE_WIS_W = `LOG2UP(ISSUE_RATIO); - localparam ISSUE_ADDRW = `LOG2UP(`NUM_REGS * (ISSUE_RATIO)); - + localparam ISSUE_WIS = `CLOG2(ISSUE_RATIO); + localparam ISSUE_WIS_W = `UP(ISSUE_WIS); + `IGNORE_UNUSED_BEGIN - function logic [ISSUE_IDX_W-1:0] wid_to_isw( + function logic [`NW_WIDTH-1:0] wis_to_wid( + input logic [ISSUE_WIS_W-1:0] wis, + input logic [ISSUE_ISW_W-1:0] isw + ); + if (ISSUE_WIS == 0) begin + wis_to_wid = `NW_WIDTH'(isw); + end else if (ISSUE_ISW == 0) begin + wis_to_wid = `NW_WIDTH'(wis); + end else begin + wis_to_wid = `NW_WIDTH'({wis, isw}); + end + endfunction + + function logic [ISSUE_ISW_W-1:0] wid_to_isw( input logic [`NW_WIDTH-1:0] wid ); - if (`ISSUE_WIDTH > 1) begin - wid_to_isw = ISSUE_IDX_W'(wid); + if (ISSUE_ISW != 0) begin + wid_to_isw = wid[ISSUE_ISW_W-1:0]; end else begin wid_to_isw = 0; end endfunction -`IGNORE_UNUSED_END - - function logic [`NW_WIDTH-1:0] wis_to_wid( - input logic [ISSUE_WIS_W-1:0] wis, - input logic [ISSUE_IDX_W-1:0] isw - ); - wis_to_wid = `NW_WIDTH'({wis, isw} >> (ISSUE_IDX_W-`CLOG2(`ISSUE_WIDTH))); - endfunction function logic [ISSUE_WIS_W-1:0] wid_to_wis( input logic [`NW_WIDTH-1:0] wid ); - wid_to_wis = ISSUE_WIS_W'(wid >> `CLOG2(`ISSUE_WIDTH)); - endfunction - - function logic [ISSUE_ADDRW-1:0] wis_to_addr( - input logic [`NR_BITS-1:0] rid, - input logic [ISSUE_WIS_W-1:0] wis - ); - wis_to_addr = ISSUE_ADDRW'({rid, wis} >> (ISSUE_WIS_W-`CLOG2(ISSUE_RATIO))); + if (ISSUE_WIS != 0) begin + wid_to_wis = ISSUE_WIS_W'(wid >> ISSUE_ISW); + end else begin + wid_to_wis = 0; + end endfunction +`IGNORE_UNUSED_END endpackage diff --git a/hw/rtl/VX_platform.vh b/hw/rtl/VX_platform.vh index c9f42cc1..ae78882b 100644 --- a/hw/rtl/VX_platform.vh +++ b/hw/rtl/VX_platform.vh @@ -14,7 +14,7 @@ `ifndef VX_PLATFORM_VH `define VX_PLATFORM_VH -`ifndef SYNTHESIS +`ifdef SV_DPI `include "util_dpi.vh" `endif diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index 1e61fdff..563cf5ba 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -65,58 +65,11 @@ module VX_socket import VX_gpu_pkg::*; #( `ifdef PERF_ENABLE VX_mem_perf_if mem_perf_tmp_if(); - cache_perf_t perf_icache; - cache_perf_t perf_dcache; - - assign mem_perf_tmp_if.icache = perf_icache; - assign mem_perf_tmp_if.dcache = perf_dcache; assign mem_perf_tmp_if.l2cache = mem_perf_if.l2cache; assign mem_perf_tmp_if.l3cache = mem_perf_if.l3cache; assign mem_perf_tmp_if.smem = 'x; assign mem_perf_tmp_if.mem = mem_perf_if.mem; -`endif - - VX_mem_bus_if #( - .DATA_SIZE (ICACHE_LINE_SIZE), - .TAG_WIDTH (ICACHE_MEM_TAG_WIDTH) - ) icache_mem_bus_if(); - - VX_mem_bus_if #( - .DATA_SIZE (DCACHE_LINE_SIZE), - .TAG_WIDTH (DCACHE_MEM_TAG_WIDTH) - ) dcache_mem_bus_if(); - - VX_mem_bus_if #( - .DATA_SIZE (`L1_LINE_SIZE), - .TAG_WIDTH (L1_MEM_TAG_WIDTH) - ) cache_mem_bus_if[2](); - - VX_mem_bus_if #( - .DATA_SIZE (`L1_LINE_SIZE), - .TAG_WIDTH (L1_MEM_ARB_TAG_WIDTH) - ) mem_bus_tmp_if[1](); - - `ASSIGN_VX_MEM_BUS_IF_X (cache_mem_bus_if[0], icache_mem_bus_if, L1_MEM_TAG_WIDTH, ICACHE_MEM_TAG_WIDTH); - `ASSIGN_VX_MEM_BUS_IF_X (cache_mem_bus_if[1], dcache_mem_bus_if, L1_MEM_TAG_WIDTH, DCACHE_MEM_TAG_WIDTH); - - `RESET_RELAY (mem_arb_reset, reset); - - VX_mem_arb #( - .NUM_INPUTS (2), - .DATA_SIZE (`L1_LINE_SIZE), - .TAG_WIDTH (L1_MEM_TAG_WIDTH), - .TAG_SEL_IDX (1), // Skip 0 for NC flag - .ARBITER ("R"), - .OUT_REG_REQ (2), - .OUT_REG_RSP (2) - ) mem_arb ( - .clk (clk), - .reset (mem_arb_reset), - .bus_in_if (cache_mem_bus_if), - .bus_out_if (mem_bus_tmp_if) - ); - - `ASSIGN_VX_MEM_BUS_IF (mem_bus_if, mem_bus_tmp_if[0]); +`endif /////////////////////////////////////////////////////////////////////////// @@ -125,6 +78,11 @@ module VX_socket import VX_gpu_pkg::*; #( .TAG_WIDTH (ICACHE_TAG_WIDTH) ) per_core_icache_bus_if[`SOCKET_SIZE](); + VX_mem_bus_if #( + .DATA_SIZE (ICACHE_LINE_SIZE), + .TAG_WIDTH (ICACHE_MEM_TAG_WIDTH) + ) icache_mem_bus_if(); + `RESET_RELAY (icache_reset, reset); VX_cache_cluster #( @@ -149,7 +107,7 @@ module VX_socket import VX_gpu_pkg::*; #( .MEM_OUT_REG (2) ) icache ( `ifdef PERF_ENABLE - .cache_perf (perf_icache), + .cache_perf (mem_perf_tmp_if.icache), `endif .clk (clk), .reset (icache_reset), @@ -160,9 +118,14 @@ module VX_socket import VX_gpu_pkg::*; #( /////////////////////////////////////////////////////////////////////////// VX_mem_bus_if #( - .DATA_SIZE (DCACHE_WORD_SIZE), + .DATA_SIZE (DCACHE_WORD_SIZE), .TAG_WIDTH (DCACHE_NOSM_TAG_WIDTH) ) per_core_dcache_bus_if[`SOCKET_SIZE * DCACHE_NUM_REQS](); + + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_LINE_SIZE), + .TAG_WIDTH (DCACHE_MEM_TAG_WIDTH) + ) dcache_mem_bus_if(); `RESET_RELAY (dcache_reset, reset); @@ -189,7 +152,7 @@ module VX_socket import VX_gpu_pkg::*; #( .MEM_OUT_REG (2) ) dcache ( `ifdef PERF_ENABLE - .cache_perf (perf_dcache), + .cache_perf (mem_perf_tmp_if.dcache), `endif .clk (clk), .reset (dcache_reset), @@ -197,6 +160,40 @@ module VX_socket import VX_gpu_pkg::*; #( .mem_bus_if (dcache_mem_bus_if) ); + /////////////////////////////////////////////////////////////////////////// + + VX_mem_bus_if #( + .DATA_SIZE (`L1_LINE_SIZE), + .TAG_WIDTH (L1_MEM_TAG_WIDTH) + ) l1_mem_bus_if[2](); + + VX_mem_bus_if #( + .DATA_SIZE (`L1_LINE_SIZE), + .TAG_WIDTH (L1_MEM_ARB_TAG_WIDTH) + ) l1_mem_arb_bus_if[1](); + + `ASSIGN_VX_MEM_BUS_IF_X (l1_mem_bus_if[0], icache_mem_bus_if, L1_MEM_TAG_WIDTH, ICACHE_MEM_TAG_WIDTH); + `ASSIGN_VX_MEM_BUS_IF_X (l1_mem_bus_if[1], dcache_mem_bus_if, L1_MEM_TAG_WIDTH, DCACHE_MEM_TAG_WIDTH); + + `RESET_RELAY (mem_arb_reset, reset); + + VX_mem_arb #( + .NUM_INPUTS (2), + .DATA_SIZE (`L1_LINE_SIZE), + .TAG_WIDTH (L1_MEM_TAG_WIDTH), + .TAG_SEL_IDX (1), // Skip 0 for NC flag + .ARBITER ("R"), + .OUT_REG_REQ (2), + .OUT_REG_RSP (2) + ) mem_arb ( + .clk (clk), + .reset (mem_arb_reset), + .bus_in_if (l1_mem_bus_if), + .bus_out_if (l1_mem_arb_bus_if) + ); + + `ASSIGN_VX_MEM_BUS_IF (mem_bus_if, l1_mem_arb_bus_if[0]); + /////////////////////////////////////////////////////////////////////////// wire [`SOCKET_SIZE-1:0] per_core_sim_ebreak; @@ -245,6 +242,6 @@ module VX_socket import VX_gpu_pkg::*; #( ); end - `BUFFER_BUSY (busy, (| per_core_busy), (`SOCKET_SIZE > 1)); + `BUFFER_EX(busy, (| per_core_busy), 1'b1, (`SOCKET_SIZE > 1)); endmodule diff --git a/hw/rtl/VX_types.vh b/hw/rtl/VX_types.vh index 388dc258..9e875d21 100644 --- a/hw/rtl/VX_types.vh +++ b/hw/rtl/VX_types.vh @@ -58,6 +58,8 @@ `define VX_CSR_MPM_BASE 12'hB00 `define VX_CSR_MPM_BASE_H 12'hB80 +`define VX_CSR_MPM_USER 12'hB03 +`define VX_CSR_MPM_USER_H 12'hB83 // Machine Performance-monitoring core counters // PERF: Standard @@ -68,29 +70,38 @@ `define VX_CSR_MINSTRET 12'hB02 `define VX_CSR_MINSTRET_H 12'hB82 // PERF: pipeline -`define VX_CSR_MPM_IBUF_ST 12'hB03 -`define VX_CSR_MPM_IBUF_ST_H 12'hB83 -`define VX_CSR_MPM_SCRB_ST 12'hB04 -`define VX_CSR_MPM_SCRB_ST_H 12'hB84 -`define VX_CSR_MPM_ALU_ST 12'hB05 -`define VX_CSR_MPM_ALU_ST_H 12'hB85 -`define VX_CSR_MPM_LSU_ST 12'hB06 -`define VX_CSR_MPM_LSU_ST_H 12'hB86 -`define VX_CSR_MPM_FPU_ST 12'hB07 -`define VX_CSR_MPM_FPU_ST_H 12'hB87 -`define VX_CSR_MPM_SFU_ST 12'hB08 -`define VX_CSR_MPM_SFU_ST_H 12'hB88 +`define VX_CSR_MPM_SCHED_ID 12'hB03 +`define VX_CSR_MPM_SCHED_ID_H 12'hB83 +`define VX_CSR_MPM_SCHED_ST 12'hB04 +`define VX_CSR_MPM_SCHED_ST_H 12'hB84 +`define VX_CSR_MPM_IBUF_ST 12'hB05 +`define VX_CSR_MPM_IBUF_ST_H 12'hB85 +`define VX_CSR_MPM_SCRB_ST 12'hB06 +`define VX_CSR_MPM_SCRB_ST_H 12'hB86 +`define VX_CSR_MPM_SCRB_ALU 12'hB07 +`define VX_CSR_MPM_SCRB_ALU_H 12'hB87 +`define VX_CSR_MPM_SCRB_FPU 12'hB08 +`define VX_CSR_MPM_SCRB_FPU_H 12'hB88 +`define VX_CSR_MPM_SCRB_LSU 12'hB09 +`define VX_CSR_MPM_SCRB_LSU_H 12'hB89 +`define VX_CSR_MPM_SCRB_SFU 12'hB0A +`define VX_CSR_MPM_SCRB_SFU_H 12'hB8A // PERF: memory -`define VX_CSR_MPM_IFETCHES 12'hB0A -`define VX_CSR_MPM_IFETCHES_H 12'hB8A -`define VX_CSR_MPM_LOADS 12'hB0B -`define VX_CSR_MPM_LOADS_H 12'hB8B -`define VX_CSR_MPM_STORES 12'hB0C -`define VX_CSR_MPM_STORES_H 12'hB8C -`define VX_CSR_MPM_IFETCH_LAT 12'hB0D -`define VX_CSR_MPM_IFETCH_LAT_H 12'hB8D -`define VX_CSR_MPM_LOAD_LAT 12'hB0E -`define VX_CSR_MPM_LOAD_LAT_H 12'hB8E +`define VX_CSR_MPM_IFETCHES 12'hB0B +`define VX_CSR_MPM_IFETCHES_H 12'hB8B +`define VX_CSR_MPM_LOADS 12'hB0C +`define VX_CSR_MPM_LOADS_H 12'hB8C +`define VX_CSR_MPM_STORES 12'hB0D +`define VX_CSR_MPM_STORES_H 12'hB8D +`define VX_CSR_MPM_IFETCH_LT 12'hB0E +`define VX_CSR_MPM_IFETCH_LT_H 12'hB8E +`define VX_CSR_MPM_LOAD_LT 12'hB0F +`define VX_CSR_MPM_LOAD_LT_H 12'hB8F +// SFU: scoreboard +`define VX_CSR_MPM_SCRB_WCTL 12'hB10 +`define VX_CSR_MPM_SCRB_WCTL_H 12'hB90 +`define VX_CSR_MPM_SCRB_CSRS 12'hB11 +`define VX_CSR_MPM_SCRB_CSRS_H 12'hB91 // Machine Performance-monitoring memory counters // PERF: icache @@ -98,59 +109,61 @@ `define VX_CSR_MPM_ICACHE_READS_H 12'hB83 `define VX_CSR_MPM_ICACHE_MISS_R 12'hB04 // read misses `define VX_CSR_MPM_ICACHE_MISS_R_H 12'hB84 +`define VX_CSR_MPM_ICACHE_MSHR_ST 12'hB05 // MSHR stalls +`define VX_CSR_MPM_ICACHE_MSHR_ST_H 12'hB85 // PERF: dcache -`define VX_CSR_MPM_DCACHE_READS 12'hB05 // total reads -`define VX_CSR_MPM_DCACHE_READS_H 12'hB85 -`define VX_CSR_MPM_DCACHE_WRITES 12'hB06 // total writes -`define VX_CSR_MPM_DCACHE_WRITES_H 12'hB86 -`define VX_CSR_MPM_DCACHE_MISS_R 12'hB07 // read misses -`define VX_CSR_MPM_DCACHE_MISS_R_H 12'hB87 -`define VX_CSR_MPM_DCACHE_MISS_W 12'hB08 // write misses -`define VX_CSR_MPM_DCACHE_MISS_W_H 12'hB88 -`define VX_CSR_MPM_DCACHE_BANK_ST 12'hB09 // bank conflicts -`define VX_CSR_MPM_DCACHE_BANK_ST_H 12'hB89 -`define VX_CSR_MPM_DCACHE_MSHR_ST 12'hB0A // MSHR stalls -`define VX_CSR_MPM_DCACHE_MSHR_ST_H 12'hB8A -// PERF: smem -`define VX_CSR_MPM_SMEM_READS 12'hB0B // memory reads -`define VX_CSR_MPM_SMEM_READS_H 12'hB8B -`define VX_CSR_MPM_SMEM_WRITES 12'hB0C // memory writes -`define VX_CSR_MPM_SMEM_WRITES_H 12'hB8C -`define VX_CSR_MPM_SMEM_BANK_ST 12'hB0D // bank conflicts -`define VX_CSR_MPM_SMEM_BANK_ST_H 12'hB8D +`define VX_CSR_MPM_DCACHE_READS 12'hB06 // total reads +`define VX_CSR_MPM_DCACHE_READS_H 12'hB86 +`define VX_CSR_MPM_DCACHE_WRITES 12'hB07 // total writes +`define VX_CSR_MPM_DCACHE_WRITES_H 12'hB87 +`define VX_CSR_MPM_DCACHE_MISS_R 12'hB08 // read misses +`define VX_CSR_MPM_DCACHE_MISS_R_H 12'hB88 +`define VX_CSR_MPM_DCACHE_MISS_W 12'hB09 // write misses +`define VX_CSR_MPM_DCACHE_MISS_W_H 12'hB89 +`define VX_CSR_MPM_DCACHE_BANK_ST 12'hB0A // bank conflicts +`define VX_CSR_MPM_DCACHE_BANK_ST_H 12'hB8A +`define VX_CSR_MPM_DCACHE_MSHR_ST 12'hB0B // MSHR stalls +`define VX_CSR_MPM_DCACHE_MSHR_ST_H 12'hB8B // PERF: l2cache -`define VX_CSR_MPM_L2CACHE_READS 12'hB0E // total reads -`define VX_CSR_MPM_L2CACHE_READS_H 12'hB8E -`define VX_CSR_MPM_L2CACHE_WRITES 12'hB0F // total writes -`define VX_CSR_MPM_L2CACHE_WRITES_H 12'hB8F -`define VX_CSR_MPM_L2CACHE_MISS_R 12'hB10 // read misses -`define VX_CSR_MPM_L2CACHE_MISS_R_H 12'hB90 -`define VX_CSR_MPM_L2CACHE_MISS_W 12'hB11 // write misses -`define VX_CSR_MPM_L2CACHE_MISS_W_H 12'hB91 -`define VX_CSR_MPM_L2CACHE_BANK_ST 12'hB12 // bank conflicts -`define VX_CSR_MPM_L2CACHE_BANK_ST_H 12'hB92 -`define VX_CSR_MPM_L2CACHE_MSHR_ST 12'hB13 // MSHR stalls -`define VX_CSR_MPM_L2CACHE_MSHR_ST_H 12'hB93 +`define VX_CSR_MPM_L2CACHE_READS 12'hB0C // total reads +`define VX_CSR_MPM_L2CACHE_READS_H 12'hB8C +`define VX_CSR_MPM_L2CACHE_WRITES 12'hB0D // total writes +`define VX_CSR_MPM_L2CACHE_WRITES_H 12'hB8D +`define VX_CSR_MPM_L2CACHE_MISS_R 12'hB0E // read misses +`define VX_CSR_MPM_L2CACHE_MISS_R_H 12'hB8E +`define VX_CSR_MPM_L2CACHE_MISS_W 12'hB0F // write misses +`define VX_CSR_MPM_L2CACHE_MISS_W_H 12'hB8F +`define VX_CSR_MPM_L2CACHE_BANK_ST 12'hB10 // bank conflicts +`define VX_CSR_MPM_L2CACHE_BANK_ST_H 12'hB90 +`define VX_CSR_MPM_L2CACHE_MSHR_ST 12'hB11 // MSHR stalls +`define VX_CSR_MPM_L2CACHE_MSHR_ST_H 12'hB91 // PERF: l3cache -`define VX_CSR_MPM_L3CACHE_READS 12'hB14 // total reads -`define VX_CSR_MPM_L3CACHE_READS_H 12'hB94 -`define VX_CSR_MPM_L3CACHE_WRITES 12'hB15 // total writes -`define VX_CSR_MPM_L3CACHE_WRITES_H 12'hB95 -`define VX_CSR_MPM_L3CACHE_MISS_R 12'hB16 // read misses -`define VX_CSR_MPM_L3CACHE_MISS_R_H 12'hB96 -`define VX_CSR_MPM_L3CACHE_MISS_W 12'hB17 // write misses -`define VX_CSR_MPM_L3CACHE_MISS_W_H 12'hB97 -`define VX_CSR_MPM_L3CACHE_BANK_ST 12'hB18 // bank conflicts -`define VX_CSR_MPM_L3CACHE_BANK_ST_H 12'hB98 -`define VX_CSR_MPM_L3CACHE_MSHR_ST 12'hB19 // MSHR stalls -`define VX_CSR_MPM_L3CACHE_MSHR_ST_H 12'hB99 +`define VX_CSR_MPM_L3CACHE_READS 12'hB12 // total reads +`define VX_CSR_MPM_L3CACHE_READS_H 12'hB92 +`define VX_CSR_MPM_L3CACHE_WRITES 12'hB13 // total writes +`define VX_CSR_MPM_L3CACHE_WRITES_H 12'hB93 +`define VX_CSR_MPM_L3CACHE_MISS_R 12'hB14 // read misses +`define VX_CSR_MPM_L3CACHE_MISS_R_H 12'hB94 +`define VX_CSR_MPM_L3CACHE_MISS_W 12'hB15 // write misses +`define VX_CSR_MPM_L3CACHE_MISS_W_H 12'hB95 +`define VX_CSR_MPM_L3CACHE_BANK_ST 12'hB16 // bank conflicts +`define VX_CSR_MPM_L3CACHE_BANK_ST_H 12'hB96 +`define VX_CSR_MPM_L3CACHE_MSHR_ST 12'hB17 // MSHR stalls +`define VX_CSR_MPM_L3CACHE_MSHR_ST_H 12'hB97 // PERF: memory -`define VX_CSR_MPM_MEM_READS 12'hB1A // total reads -`define VX_CSR_MPM_MEM_READS_H 12'hB9A -`define VX_CSR_MPM_MEM_WRITES 12'hB1B // total writes -`define VX_CSR_MPM_MEM_WRITES_H 12'hB9B -`define VX_CSR_MPM_MEM_LAT 12'hB1C // memory latency -`define VX_CSR_MPM_MEM_LAT_H 12'hB9C +`define VX_CSR_MPM_MEM_READS 12'hB18 // total reads +`define VX_CSR_MPM_MEM_READS_H 12'hB98 +`define VX_CSR_MPM_MEM_WRITES 12'hB19 // total writes +`define VX_CSR_MPM_MEM_WRITES_H 12'hB99 +`define VX_CSR_MPM_MEM_LT 12'hB1A // memory latency +`define VX_CSR_MPM_MEM_LT_H 12'hB9A +// PERF: smem +`define VX_CSR_MPM_SMEM_READS 12'hB1B // memory reads +`define VX_CSR_MPM_SMEM_READS_H 12'hB9B +`define VX_CSR_MPM_SMEM_WRITES 12'hB1C // memory writes +`define VX_CSR_MPM_SMEM_WRITES_H 12'hB9C +`define VX_CSR_MPM_SMEM_BANK_ST 12'hB1D // bank conflicts +`define VX_CSR_MPM_SMEM_BANK_ST_H 12'hB9D // Machine Information Registers diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 5bd628d5..a955c8f5 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -22,15 +22,15 @@ module Vortex import VX_gpu_pkg::*; ( // Memory request output wire mem_req_valid, - output wire mem_req_rw, - output wire [`VX_MEM_BYTEEN_WIDTH-1:0] mem_req_byteen, + output wire mem_req_rw, + output wire [`VX_MEM_BYTEEN_WIDTH-1:0] mem_req_byteen, output wire [`VX_MEM_ADDR_WIDTH-1:0] mem_req_addr, output wire [`VX_MEM_DATA_WIDTH-1:0] mem_req_data, output wire [`VX_MEM_TAG_WIDTH-1:0] mem_req_tag, input wire mem_req_ready, // Memory response - input wire mem_rsp_valid, + input wire mem_rsp_valid, input wire [`VX_MEM_DATA_WIDTH-1:0] mem_rsp_data, input wire [`VX_MEM_TAG_WIDTH-1:0] mem_rsp_tag, output wire mem_rsp_ready, @@ -45,17 +45,11 @@ module Vortex import VX_gpu_pkg::*; ( ); `ifdef PERF_ENABLE - VX_mem_perf_if mem_perf_if(); - cache_perf_t perf_l3cache; - mem_perf_t mem_perf; - - assign mem_perf_if.icache = 'x; - assign mem_perf_if.dcache = 'x; + VX_mem_perf_if mem_perf_if(); + assign mem_perf_if.icache = 'x; + assign mem_perf_if.dcache = 'x; assign mem_perf_if.l2cache = 'x; - assign mem_perf_if.l3cache = perf_l3cache; - assign mem_perf_if.smem = 'x; - assign mem_perf_if.mem = mem_perf; -`endif +`endif VX_mem_bus_if #( .DATA_SIZE (`L2_LINE_SIZE), @@ -93,7 +87,7 @@ module Vortex import VX_gpu_pkg::*; ( .reset (l3_reset), `ifdef PERF_ENABLE - .cache_perf (perf_l3cache), + .cache_perf (mem_perf_if.l3cache), `endif .core_bus_if (per_cluster_mem_bus_if), @@ -166,11 +160,12 @@ module Vortex import VX_gpu_pkg::*; ( ); end - `BUFFER_BUSY (busy, (| per_cluster_busy), (`NUM_CLUSTERS > 1)); + `BUFFER_EX(busy, (| per_cluster_busy), 1'b1, (`NUM_CLUSTERS > 1)); `ifdef PERF_ENABLE - reg [`PERF_CTR_BITS-1:0] perf_mem_pending_reads; + reg [`PERF_CTR_BITS-1:0] perf_mem_pending_reads; + mem_perf_t mem_perf; always @(posedge clk) begin if (reset) begin @@ -181,19 +176,19 @@ module Vortex import VX_gpu_pkg::*; ( end end + wire mem_rd_req_fire = mem_req_fire && ~mem_bus_if.req_data.rw; + wire mem_wr_req_fire = mem_req_fire && mem_bus_if.req_data.rw; + always @(posedge clk) begin if (reset) begin mem_perf <= '0; - end else begin - if (mem_req_fire && ~mem_bus_if.req_data.rw) begin - mem_perf.reads <= mem_perf.reads + `PERF_CTR_BITS'(1); - end - if (mem_req_fire && mem_bus_if.req_data.rw) begin - mem_perf.writes <= mem_perf.writes + `PERF_CTR_BITS'(1); - end + end else begin + mem_perf.reads <= mem_perf.reads + `PERF_CTR_BITS'(mem_rd_req_fire); + mem_perf.writes <= mem_perf.writes + `PERF_CTR_BITS'(mem_wr_req_fire); mem_perf.latency <= mem_perf.latency + perf_mem_pending_reads; end end + assign mem_perf_if.mem = mem_perf; `endif diff --git a/hw/rtl/afu/xrt/VX_afu_wrap.sv b/hw/rtl/afu/xrt/VX_afu_wrap.sv index 3c4b3947..2abbbe43 100644 --- a/hw/rtl/afu/xrt/VX_afu_wrap.sv +++ b/hw/rtl/afu/xrt/VX_afu_wrap.sv @@ -262,7 +262,7 @@ module VX_afu_wrap #( .m_axi_awready (m_axi_mem_awready_a), .m_axi_awaddr (m_axi_mem_awaddr_w), .m_axi_awid (m_axi_mem_awid_a), - `UNUSED_PIN (m_axi_awlen), + .m_axi_awlen (m_axi_mem_awlen_a), `UNUSED_PIN (m_axi_awsize), `UNUSED_PIN (m_axi_awburst), `UNUSED_PIN (m_axi_awlock), diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 30594f1a..891512da 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -530,14 +530,17 @@ module VX_cache import VX_gpu_pkg::*; #( wire [`CLOG2(NUM_REQS+1)-1:0] perf_core_reads_per_cycle; wire [`CLOG2(NUM_REQS+1)-1:0] perf_core_writes_per_cycle; - wire [NUM_REQS-1:0] perf_core_reads_per_req = core_req_valid & core_req_ready & ~core_req_rw; - wire [NUM_REQS-1:0] perf_core_writes_per_req = core_req_valid & core_req_ready & core_req_rw; + wire [NUM_REQS-1:0] perf_core_reads_per_req; + wire [NUM_REQS-1:0] perf_core_writes_per_req; // per cycle: read misses, write misses, msrq stalls, pipeline stalls wire [`CLOG2(NUM_BANKS+1)-1:0] perf_read_miss_per_cycle; wire [`CLOG2(NUM_BANKS+1)-1:0] perf_write_miss_per_cycle; wire [`CLOG2(NUM_BANKS+1)-1:0] perf_mshr_stall_per_cycle; - wire [`CLOG2(NUM_BANKS+1)-1:0] perf_crsp_stall_per_cycle; + wire [`CLOG2(NUM_REQS+1)-1:0] perf_crsp_stall_per_cycle; + + `BUFFER(perf_core_reads_per_req, core_req_valid & core_req_ready & ~core_req_rw); + `BUFFER(perf_core_writes_per_req, core_req_valid & core_req_ready & core_req_rw); `POP_COUNT(perf_core_reads_per_cycle, perf_core_reads_per_req); `POP_COUNT(perf_core_writes_per_cycle, perf_core_writes_per_req); @@ -560,7 +563,7 @@ module VX_cache import VX_gpu_pkg::*; #( reg [`PERF_CTR_BITS-1:0] perf_write_misses; reg [`PERF_CTR_BITS-1:0] perf_mshr_stalls; reg [`PERF_CTR_BITS-1:0] perf_mem_stalls; - reg [`PERF_CTR_BITS-1:0] perf_crsp_stalls; + reg [`PERF_CTR_BITS-1:0] perf_crsp_stalls; always @(posedge clk) begin if (reset) begin diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 18e26eb2..46447e4f 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -83,8 +83,9 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( `STATIC_ASSERT(NUM_INPUTS >= NUM_CACHES, ("invalid parameter")) `ifdef PERF_ENABLE - cache_perf_t perf_cache_unit[NUM_CACHES]; - `PERF_CACHE_REDUCE (cache_perf, perf_cache_unit, NUM_CACHES); + cache_perf_t perf_cache_tmp[1], perf_cache_unit[NUM_CACHES]; + `PERF_CACHE_ADD (perf_cache_tmp, perf_cache_unit, 1, NUM_CACHES) + assign cache_perf = perf_cache_tmp[0]; `endif VX_mem_bus_if #( diff --git a/hw/rtl/cache/VX_cache_cluster_top.sv b/hw/rtl/cache/VX_cache_cluster_top.sv deleted file mode 100644 index 500f2c87..00000000 --- a/hw/rtl/cache/VX_cache_cluster_top.sv +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_cache_define.vh" - -module VX_cache_cluster_top import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - - parameter NUM_UNITS = 2, - parameter NUM_INPUTS = 4, - parameter TAG_SEL_IDX = 0, - - // Number of Word requests per cycle - parameter NUM_REQS = 4, - - // Size of cache in bytes - parameter CACHE_SIZE = 16384, - // Size of line inside a bank in bytes - parameter LINE_SIZE = 16, - // Number of banks - parameter NUM_BANKS = 4, - // Number of associative ways - parameter NUM_WAYS = 4, - // Size of a word in bytes - parameter WORD_SIZE = 4, - - // Core Response Queue Size - parameter CRSQ_SIZE = 2, - // Miss Reserv Queue Knob - parameter MSHR_SIZE = 16, - // Memory Response Queue Size - parameter MRSQ_SIZE = 0, - // Memory Request Queue Size - parameter MREQ_SIZE = 4, - - // Enable cache writeable - parameter WRITE_ENABLE = 1, - - // Request debug identifier - parameter UUID_WIDTH = 0, - - // core request tag size - parameter TAG_WIDTH = UUID_WIDTH + 16, - - // enable bypass for non-cacheable addresses - parameter NC_ENABLE = 1, - - // Core response output register - parameter CORE_OUT_REG = 2, - - // Memory request output register - parameter MEM_OUT_REG = 2, - - parameter NUM_CACHES = `UP(NUM_UNITS), - parameter PASSTHRU = (NUM_UNITS == 0), - parameter ARB_TAG_WIDTH = TAG_WIDTH + `ARB_SEL_BITS(NUM_INPUTS, NUM_CACHES), - parameter MEM_TAG_WIDTH = PASSTHRU ? (NC_ENABLE ? `CACHE_NC_BYPASS_TAG_WIDTH(NUM_REQS, LINE_SIZE, WORD_SIZE, ARB_TAG_WIDTH) : - `CACHE_BYPASS_TAG_WIDTH(NUM_REQS, LINE_SIZE, WORD_SIZE, ARB_TAG_WIDTH)) : - (NC_ENABLE ? `CACHE_NC_MEM_TAG_WIDTH(MSHR_SIZE, NUM_BANKS, NUM_REQS, LINE_SIZE, WORD_SIZE, ARB_TAG_WIDTH) : - `CACHE_MEM_TAG_WIDTH(MSHR_SIZE, NUM_BANKS)), - parameter MEM_TAG_X_WIDTH = MEM_TAG_WIDTH + `ARB_SEL_BITS(NUM_CACHES, 1) - ) ( - input wire clk, - input wire reset, - -// PERF -`ifdef PERF_ENABLE - output cache_perf_t cache_perf, -`endif - - // Core request - input wire [NUM_INPUTS-1:0][NUM_REQS-1:0] core_req_valid, - input wire [NUM_INPUTS-1:0][NUM_REQS-1:0] core_req_rw, - input wire [NUM_INPUTS-1:0][NUM_REQS-1:0][WORD_SIZE-1:0] core_req_byteen, - input wire [NUM_INPUTS-1:0][NUM_REQS-1:0][`CS_WORD_ADDR_WIDTH-1:0] core_req_addr, - input wire [NUM_INPUTS-1:0][NUM_REQS-1:0][`CS_WORD_WIDTH-1:0] core_req_data, - input wire [NUM_INPUTS-1:0][NUM_REQS-1:0][TAG_WIDTH-1:0] core_req_tag, - output wire [NUM_INPUTS-1:0][NUM_REQS-1:0] core_req_ready, - - // Core response - output wire [NUM_INPUTS-1:0][NUM_REQS-1:0] core_rsp_valid, - output wire [NUM_INPUTS-1:0][NUM_REQS-1:0][`CS_WORD_WIDTH-1:0] core_rsp_data, - output wire [NUM_INPUTS-1:0][NUM_REQS-1:0][TAG_WIDTH-1:0] core_rsp_tag, - input wire [NUM_INPUTS-1:0][NUM_REQS-1:0] core_rsp_ready, - - // Memory request - output wire mem_req_valid, - output wire mem_req_rw, - output wire [LINE_SIZE-1:0] mem_req_byteen, - output wire [`CS_MEM_ADDR_WIDTH-1:0] mem_req_addr, - output wire [`CS_LINE_WIDTH-1:0] mem_req_data, - output wire [MEM_TAG_X_WIDTH-1:0] mem_req_tag, - input wire mem_req_ready, - - // Memory response - input wire mem_rsp_valid, - input wire [`CS_LINE_WIDTH-1:0] mem_rsp_data, - input wire [MEM_TAG_X_WIDTH-1:0] mem_rsp_tag, - output wire mem_rsp_ready -); - VX_mem_bus_if #( - .DATA_SIZE (WORD_SIZE), - .TAG_WIDTH (TAG_WIDTH) - ) core_bus_if[NUM_INPUTS * NUM_REQS](); - - VX_mem_bus_if #( - .DATA_SIZE (LINE_SIZE), - .TAG_WIDTH (MEM_TAG_X_WIDTH) - ) mem_bus_if(); - - // Core request - for (genvar i = 0; i < NUM_INPUTS; ++i) begin - for (genvar r = 0; r < NUM_REQS; ++r) begin - assign core_bus_if[i * NUM_REQS + r].req_valid = core_req_valid[i][r]; - assign core_bus_if[i * NUM_REQS + r].req_data.rw = core_req_rw[i][r]; - assign core_bus_if[i * NUM_REQS + r].req_data.byteen = core_req_byteen[i][r]; - assign core_bus_if[i * NUM_REQS + r].req_data.addr = core_req_addr[i][r]; - assign core_bus_if[i * NUM_REQS + r].req_data.data = core_req_data[i][r]; - assign core_bus_if[i * NUM_REQS + r].req_data.tag = core_req_tag[i][r]; - assign core_req_ready[i][r] = core_bus_if[i * NUM_REQS + r].req_ready; - end - end - - // Core response - for (genvar i = 0; i < NUM_INPUTS; ++i) begin - for (genvar r = 0; r < NUM_REQS; ++r) begin - assign core_rsp_valid[i][r] = core_bus_if[i * NUM_REQS + r].rsp_valid; - assign core_rsp_data[i][r] = core_bus_if[i * NUM_REQS + r].rsp_data.data; - assign core_rsp_tag[i][r] = core_bus_if[i * NUM_REQS + r].rsp_data.tag; - assign core_bus_if[i * NUM_REQS + r].rsp_ready = core_rsp_ready[i][r]; - end - end - - // Memory request - assign mem_req_valid = mem_bus_if.req_valid; - assign mem_req_rw = mem_bus_if.req_data.rw; - assign mem_req_byteen = mem_bus_if.req_data.byteen; - assign mem_req_addr = mem_bus_if.req_data.addr; - assign mem_req_data = mem_bus_if.req_data.data; - assign mem_req_tag = mem_bus_if.req_data.tag; - assign mem_bus_if.req_ready = mem_req_ready; - - // Memory response - assign mem_bus_if.rsp_valid = mem_rsp_valid; - assign mem_bus_if.rsp_data.data = mem_rsp_data; - assign mem_bus_if.rsp_data.tag = mem_rsp_tag; - assign mem_rsp_ready = mem_bus_if.rsp_ready; - - VX_cache_cluster #( - .INSTANCE_ID (INSTANCE_ID), - .NUM_UNITS (NUM_UNITS), - .NUM_INPUTS (NUM_INPUTS), - .TAG_SEL_IDX (TAG_SEL_IDX), - .NUM_REQS (NUM_REQS), - .CACHE_SIZE (CACHE_SIZE), - .LINE_SIZE (LINE_SIZE), - .NUM_BANKS (NUM_BANKS), - .NUM_WAYS (NUM_WAYS), - .WORD_SIZE (WORD_SIZE), - .CRSQ_SIZE (CRSQ_SIZE), - .MSHR_SIZE (MSHR_SIZE), - .MRSQ_SIZE (MRSQ_SIZE), - .MREQ_SIZE (MREQ_SIZE), - .WRITE_ENABLE (WRITE_ENABLE), - .UUID_WIDTH (UUID_WIDTH), - .TAG_WIDTH (TAG_WIDTH), - .NC_ENABLE (NC_ENABLE), - .CORE_OUT_REG (CORE_OUT_REG), - .MEM_OUT_REG (MEM_OUT_REG) - ) cache ( - `ifdef PERF_ENABLE - .cache_perf (cache_perf), - `endif - .clk (clk), - .reset (reset), - .core_bus_if (core_bus_if), - .mem_bus_if (mem_bus_if) - ); - -endmodule diff --git a/hw/rtl/cache/VX_cache_data.sv b/hw/rtl/cache/VX_cache_data.sv index 70dcc6e8..5106d7d5 100644 --- a/hw/rtl/cache/VX_cache_data.sv +++ b/hw/rtl/cache/VX_cache_data.sv @@ -93,13 +93,7 @@ module VX_cache_data #( assign wren = fill; end - wire [`CLOG2(NUM_WAYS)-1:0] way_idx; - // generate if (NUM_WAYS == 1) begin - // wire [0:0] way_idx; - // end else begin - // wire [`CLOG2(NUM_WAYS)-1:0] way_idx; - // end - // endgenerate + wire [`LOG2UP(NUM_WAYS)-1:0] way_idx; VX_onehot_encoder #( .N (NUM_WAYS) diff --git a/hw/rtl/cache/VX_cache_define.vh b/hw/rtl/cache/VX_cache_define.vh index a2856432..80032e7c 100644 --- a/hw/rtl/cache/VX_cache_define.vh +++ b/hw/rtl/cache/VX_cache_define.vh @@ -63,4 +63,16 @@ `define CS_LINE_TO_FULL_ADDR(x, i) {x, (`XLEN-$bits(x))'(i << (`XLEN-$bits(x)-`CS_BANK_SEL_BITS))} `define CS_MEM_TO_FULL_ADDR(x) {x, (`XLEN-$bits(x))'(0)} +/////////////////////////////////////////////////////////////////////////////// + +`define PERF_CACHE_ADD(dst, src, dcount, scount) \ + `PERF_COUNTER_ADD (dst, src, reads, `PERF_CTR_BITS, dcount, scount, (((scount + dcount - 1) / dcount) > 1)) \ + `PERF_COUNTER_ADD (dst, src, writes, `PERF_CTR_BITS, dcount, scount, (((scount + dcount - 1) / dcount) > 1)) \ + `PERF_COUNTER_ADD (dst, src, read_misses, `PERF_CTR_BITS, dcount, scount, (((scount + dcount - 1) / dcount) > 1)) \ + `PERF_COUNTER_ADD (dst, src, write_misses, `PERF_CTR_BITS, dcount, scount, (((scount + dcount - 1) / dcount) > 1)) \ + `PERF_COUNTER_ADD (dst, src, bank_stalls, `PERF_CTR_BITS, dcount, scount, (((scount + dcount - 1) / dcount) > 1)) \ + `PERF_COUNTER_ADD (dst, src, mshr_stalls, `PERF_CTR_BITS, dcount, scount, (((scount + dcount - 1) / dcount) > 1)) \ + `PERF_COUNTER_ADD (dst, src, mem_stalls, `PERF_CTR_BITS, dcount, scount, (((scount + dcount - 1) / dcount) > 1)) \ + `PERF_COUNTER_ADD (dst, src, crsp_stalls, `PERF_CTR_BITS, dcount, scount, (((scount + dcount - 1) / dcount) > 1)) + `endif // VX_CACHE_DEFINE_VH diff --git a/hw/rtl/cache/VX_cache_top.sv b/hw/rtl/cache/VX_cache_top.sv index 9e36d9af..00b2f048 100644 --- a/hw/rtl/cache/VX_cache_top.sv +++ b/hw/rtl/cache/VX_cache_top.sv @@ -13,7 +13,7 @@ `include "VX_cache_define.vh" -module VX_cache_top #( +module VX_cache_top import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", // Number of Word requests per cycle @@ -22,7 +22,7 @@ module VX_cache_top #( // Size of cache in bytes parameter CACHE_SIZE = 16384, // Size of line inside a bank in bytes - parameter LINE_SIZE = 16, + parameter LINE_SIZE = 64, // Number of banks parameter NUM_BANKS = 4, // Number of associative ways diff --git a/hw/rtl/core/VX_commit.sv b/hw/rtl/core/VX_commit.sv index 2a32d96c..976a158f 100644 --- a/hw/rtl/core/VX_commit.sv +++ b/hw/rtl/core/VX_commit.sv @@ -45,7 +45,7 @@ module VX_commit import VX_gpu_pkg::*; #( VX_commit_if commit_if[`ISSUE_WIDTH](); - wire [`ISSUE_WIDTH-1:0] commit_fire; + wire [`ISSUE_WIDTH-1:0] commit_fire; wire [`ISSUE_WIDTH-1:0][`NW_WIDTH-1:0] commit_wid; wire [`ISSUE_WIDTH-1:0][`NUM_THREADS-1:0] commit_tmask; wire [`ISSUE_WIDTH-1:0] commit_eop; @@ -92,24 +92,24 @@ module VX_commit import VX_gpu_pkg::*; #( `UNUSED_PIN (sel_out) ); - assign commit_fire[i] = commit_if[i].valid && commit_if[i].ready; - assign commit_tmask[i] = {`NUM_THREADS{commit_fire[i]}} & commit_if[i].data.tmask; - assign commit_wid[i] = commit_if[i].data.wid; - assign commit_eop[i] = commit_if[i].data.eop; + assign commit_fire[i] = commit_if[i].valid && commit_if[i].ready; + assign commit_tmask[i]= {`NUM_THREADS{commit_fire[i]}} & commit_if[i].data.tmask; + assign commit_wid[i] = commit_if[i].data.wid; + assign commit_eop[i] = commit_if[i].data.eop; end // CSRs update wire [`ISSUE_WIDTH-1:0][COMMIT_SIZEW-1:0] commit_size, commit_size_r; - wire [COMMIT_ALL_SIZEW-1:0] commit_size_all, commit_size_all_r; + wire [COMMIT_ALL_SIZEW-1:0] commit_size_all_r, commit_size_all_rr; wire commit_fire_any, commit_fire_any_r, commit_fire_any_rr; assign commit_fire_any = (| commit_fire); for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin - wire [COMMIT_SIZEW-1:0] pop_count; - `POP_COUNT(pop_count, commit_tmask[i]); - assign commit_size[i] = pop_count; + wire [COMMIT_SIZEW-1:0] count; + `POP_COUNT(count, commit_tmask[i]); + assign commit_size[i] = count; end VX_pipe_register #( @@ -130,7 +130,7 @@ module VX_commit import VX_gpu_pkg::*; #( .OP ("+") ) commit_size_reduce ( .data_in (commit_size_r), - .data_out (commit_size_all) + .data_out (commit_size_all_r) ); VX_pipe_register #( @@ -140,26 +140,26 @@ module VX_commit import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .enable (1'b1), - .data_in ({commit_fire_any_r, commit_size_all}), - .data_out ({commit_fire_any_rr, commit_size_all_r}) + .data_in ({commit_fire_any_r, commit_size_all_r}), + .data_out ({commit_fire_any_rr, commit_size_all_rr}) ); reg [`PERF_CTR_BITS-1:0] instret; - always @(posedge clk) begin if (reset) begin instret <= '0; end else begin if (commit_fire_any_rr) begin - instret <= instret + `PERF_CTR_BITS'(commit_size_all_r); + instret <= instret + `PERF_CTR_BITS'(commit_size_all_rr); end end end - assign commit_csr_if.instret = instret; // Committed instructions + wire [`ISSUE_WIDTH-1:0] committed = commit_fire & commit_eop; + VX_pipe_register #( .DATAW (`ISSUE_WIDTH * (1 + `NW_WIDTH)), .RESETW (`ISSUE_WIDTH) @@ -167,23 +167,23 @@ module VX_commit import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .enable (1'b1), - .data_in ({(commit_fire & commit_eop), commit_wid}), + .data_in ({committed, commit_wid}), .data_out ({commit_sched_if.committed, commit_sched_if.committed_wid}) ); // Writeback for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin - assign writeback_if[i].valid = commit_if[i].valid && commit_if[i].data.wb; + assign writeback_if[i].valid = commit_if[i].valid && commit_if[i].data.wb; assign writeback_if[i].data.uuid = commit_if[i].data.uuid; - assign writeback_if[i].data.wis = wid_to_wis(commit_if[i].data.wid); - assign writeback_if[i].data.PC = commit_if[i].data.PC; - assign writeback_if[i].data.tmask = commit_if[i].data.tmask; - assign writeback_if[i].data.rd = commit_if[i].data.rd; + assign writeback_if[i].data.wis = wid_to_wis(commit_if[i].data.wid); + assign writeback_if[i].data.PC = commit_if[i].data.PC; + assign writeback_if[i].data.tmask= commit_if[i].data.tmask; + assign writeback_if[i].data.rd = commit_if[i].data.rd; assign writeback_if[i].data.data = commit_if[i].data.data; - assign writeback_if[i].data.sop = commit_if[i].data.sop; - assign writeback_if[i].data.eop = commit_if[i].data.eop; - assign commit_if[i].ready = 1'b1; + assign writeback_if[i].data.sop = commit_if[i].data.sop; + assign writeback_if[i].data.eop = commit_if[i].data.eop; + assign commit_if[i].ready = 1'b1; // writeback has no backpressure end // simulation helper signal to get RISC-V tests Pass/Fail status diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index c16ddff2..702277c6 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -1,339 +1,341 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_define.vh" - -`ifdef EXT_F_ENABLE -`include "VX_fpu_define.vh" -`endif - -module VX_core import VX_gpu_pkg::*; #( - parameter CORE_ID = 0 -) ( - `SCOPE_IO_DECL - - // Clock - input wire clk, - input wire reset, - -`ifdef PERF_ENABLE - VX_mem_perf_if.slave mem_perf_if, -`endif - - VX_dcr_bus_if.slave dcr_bus_if, - - VX_mem_bus_if.master smem_bus_if [DCACHE_NUM_REQS], - - VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], - - VX_mem_bus_if.master icache_bus_if, - -`ifdef GBAR_ENABLE - VX_gbar_bus_if.master gbar_bus_if, -`endif - - // simulation helper signals - output wire sim_ebreak, - output wire [`NUM_REGS-1:0][`XLEN-1:0] sim_wb_value, - - // Status - output wire busy -); - VX_schedule_if schedule_if(); - VX_fetch_if fetch_if(); - VX_decode_if decode_if(); - VX_sched_csr_if sched_csr_if(); - VX_decode_sched_if decode_sched_if(); - VX_commit_sched_if commit_sched_if(); - VX_commit_csr_if commit_csr_if(); - VX_branch_ctl_if branch_ctl_if[`NUM_ALU_BLOCKS](); - VX_warp_ctl_if warp_ctl_if(); - - VX_dispatch_if alu_dispatch_if[`ISSUE_WIDTH](); - VX_commit_if alu_commit_if[`ISSUE_WIDTH](); - - VX_dispatch_if lsu_dispatch_if[`ISSUE_WIDTH](); - VX_commit_if lsu_commit_if[`ISSUE_WIDTH](); -`ifdef EXT_F_ENABLE - VX_dispatch_if fpu_dispatch_if[`ISSUE_WIDTH](); - VX_commit_if fpu_commit_if[`ISSUE_WIDTH](); -`endif - VX_dispatch_if sfu_dispatch_if[`ISSUE_WIDTH](); - VX_commit_if sfu_commit_if[`ISSUE_WIDTH](); - - VX_writeback_if writeback_if[`ISSUE_WIDTH](); - - VX_mem_bus_if #( - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_TAG_WIDTH) - ) dcache_bus_tmp_if[DCACHE_NUM_REQS](); - -`ifdef PERF_ENABLE - VX_pipeline_perf_if pipeline_perf_if(); - VX_mem_perf_if mem_perf_tmp_if(); - - assign mem_perf_tmp_if.icache = mem_perf_if.icache; - assign mem_perf_tmp_if.dcache = mem_perf_if.dcache; - assign mem_perf_tmp_if.l2cache = mem_perf_if.l2cache; - assign mem_perf_tmp_if.l3cache = mem_perf_if.l3cache; -`ifdef SM_ENABLE - cache_perf_t smem_perf; - assign mem_perf_tmp_if.smem = smem_perf; -`else - assign mem_perf_tmp_if.smem = '0; -`endif - assign mem_perf_tmp_if.mem = mem_perf_if.mem; -`endif - - `RESET_RELAY (dcr_data_reset, reset); - `RESET_RELAY (schedule_reset, reset); - `RESET_RELAY (fetch_reset, reset); - `RESET_RELAY (decode_reset, reset); - `RESET_RELAY (issue_reset, reset); - `RESET_RELAY (execute_reset, reset); - `RESET_RELAY (commit_reset, reset); - - base_dcrs_t base_dcrs; - - VX_dcr_data dcr_data ( - .clk (clk), - .reset (dcr_data_reset), - .dcr_bus_if (dcr_bus_if), - .base_dcrs (base_dcrs) - ); - - `SCOPE_IO_SWITCH (3) - - VX_schedule #( - .CORE_ID (CORE_ID) - ) schedule ( - .clk (clk), - .reset (schedule_reset), - - .base_dcrs (base_dcrs), - - .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if), - .decode_sched_if(decode_sched_if), - .commit_sched_if(commit_sched_if), - - .schedule_if (schedule_if), - `ifdef GBAR_ENABLE - .gbar_bus_if (gbar_bus_if), - `endif - .sched_csr_if (sched_csr_if), - - .busy (busy) - ); - - VX_fetch #( - .CORE_ID (CORE_ID) - ) fetch ( - `SCOPE_IO_BIND (0) - .clk (clk), - .reset (fetch_reset), - .icache_bus_if (icache_bus_if), - .schedule_if (schedule_if), - .fetch_if (fetch_if) - ); - - VX_decode #( - .CORE_ID (CORE_ID) - ) decode ( - .clk (clk), - .reset (decode_reset), - .fetch_if (fetch_if), - .decode_if (decode_if), - .decode_sched_if(decode_sched_if) - ); - - VX_issue #( - .CORE_ID (CORE_ID) - ) issue ( - `SCOPE_IO_BIND (1) - - .clk (clk), - .reset (issue_reset), - - `ifdef PERF_ENABLE - .perf_issue_if (pipeline_perf_if.issue), - `endif - - .decode_if (decode_if), - .writeback_if (writeback_if), - - .alu_dispatch_if(alu_dispatch_if), - .lsu_dispatch_if(lsu_dispatch_if), - `ifdef EXT_F_ENABLE - .fpu_dispatch_if(fpu_dispatch_if), - `endif - .sfu_dispatch_if(sfu_dispatch_if) - ); - - VX_execute #( - .CORE_ID (CORE_ID) - ) execute ( - `SCOPE_IO_BIND (2) - - .clk (clk), - .reset (execute_reset), - - .base_dcrs (base_dcrs), - - `ifdef PERF_ENABLE - .mem_perf_if (mem_perf_tmp_if), - .pipeline_perf_if(pipeline_perf_if), - `endif - - .dcache_bus_if (dcache_bus_tmp_if), - - `ifdef EXT_F_ENABLE - .fpu_dispatch_if(fpu_dispatch_if), - .fpu_commit_if (fpu_commit_if), - `endif - - .commit_csr_if (commit_csr_if), - .sched_csr_if (sched_csr_if), - - .alu_dispatch_if(alu_dispatch_if), - .lsu_dispatch_if(lsu_dispatch_if), - .sfu_dispatch_if(sfu_dispatch_if), - - .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if), - - .alu_commit_if (alu_commit_if), - .lsu_commit_if (lsu_commit_if), - .sfu_commit_if (sfu_commit_if), - - .sim_ebreak (sim_ebreak) - ); - - VX_commit #( - .CORE_ID (CORE_ID) - ) commit ( - .clk (clk), - .reset (commit_reset), - - .alu_commit_if (alu_commit_if), - .lsu_commit_if (lsu_commit_if), - `ifdef EXT_F_ENABLE - .fpu_commit_if (fpu_commit_if), - `endif - .sfu_commit_if (sfu_commit_if), - - .writeback_if (writeback_if), - - .commit_csr_if (commit_csr_if), - .commit_sched_if(commit_sched_if), - - .sim_wb_value (sim_wb_value) - ); - -`ifdef SM_ENABLE - - VX_smem_unit #( - .CORE_ID (CORE_ID) - ) smem_unit ( - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .cache_perf (smem_perf), - `endif - .dcache_bus_in_if (dcache_bus_tmp_if), - .dcache_bus_out_if (dcache_bus_if), - .smem_bus_out_if (smem_bus_if) - ); - -`else - - for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin - `ASSIGN_VX_MEM_BUS_IF (dcache_bus_if[i], dcache_bus_tmp_if[i]); - end - -`endif - -`ifdef PERF_ENABLE - - wire [`CLOG2(DCACHE_NUM_REQS+1)-1:0] perf_dcache_rd_req_per_cycle; - wire [`CLOG2(DCACHE_NUM_REQS+1)-1:0] perf_dcache_wr_req_per_cycle; - - wire [`CLOG2(DCACHE_NUM_REQS+1)-1:0] perf_dcache_rsp_per_cycle; - - wire perf_icache_pending_read_cycle; - wire [`CLOG2(DCACHE_NUM_REQS+1)+1-1:0] perf_dcache_pending_read_cycle; - - reg [`PERF_CTR_BITS-1:0] perf_icache_pending_reads; - reg [`PERF_CTR_BITS-1:0] perf_dcache_pending_reads; - - reg [`PERF_CTR_BITS-1:0] perf_ifetches; - reg [`PERF_CTR_BITS-1:0] perf_loads; - reg [`PERF_CTR_BITS-1:0] perf_stores; - - wire perf_icache_req_fire = icache_bus_if.req_valid & icache_bus_if.req_ready; - wire perf_icache_rsp_fire = icache_bus_if.rsp_valid & icache_bus_if.rsp_ready; - - wire [DCACHE_NUM_REQS-1:0] perf_dcache_rd_req_fire, perf_dcache_wr_req_fire, perf_dcache_rsp_fire; - - for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin - assign perf_dcache_rd_req_fire[i] = dcache_bus_if[i].req_valid && ~dcache_bus_if[i].req_data.rw && dcache_bus_if[i].req_ready; - assign perf_dcache_wr_req_fire[i] = dcache_bus_if[i].req_valid && dcache_bus_if[i].req_data.rw && dcache_bus_if[i].req_ready; - assign perf_dcache_rsp_fire[i] = dcache_bus_if[i].rsp_valid && dcache_bus_if[i].rsp_ready; - end - - `POP_COUNT(perf_dcache_rd_req_per_cycle, perf_dcache_rd_req_fire); - `POP_COUNT(perf_dcache_wr_req_per_cycle, perf_dcache_wr_req_fire); - `POP_COUNT(perf_dcache_rsp_per_cycle, perf_dcache_rsp_fire); - - assign perf_icache_pending_read_cycle = perf_icache_req_fire - perf_icache_rsp_fire; - assign perf_dcache_pending_read_cycle = perf_dcache_rd_req_per_cycle - perf_dcache_rsp_per_cycle; - - always @(posedge clk) begin - if (reset) begin - perf_icache_pending_reads <= '0; - perf_dcache_pending_reads <= '0; - end else begin - perf_icache_pending_reads <= $signed(perf_icache_pending_reads) + `PERF_CTR_BITS'($signed(perf_icache_pending_read_cycle)); - perf_dcache_pending_reads <= $signed(perf_dcache_pending_reads) + `PERF_CTR_BITS'($signed(perf_dcache_pending_read_cycle)); - end - end - - reg [`PERF_CTR_BITS-1:0] perf_icache_lat; - reg [`PERF_CTR_BITS-1:0] perf_dcache_lat; - - always @(posedge clk) begin - if (reset) begin - perf_ifetches <= '0; - perf_loads <= '0; - perf_stores <= '0; - perf_icache_lat <= '0; - perf_dcache_lat <= '0; - end else begin - perf_ifetches <= perf_ifetches + `PERF_CTR_BITS'(perf_icache_req_fire); - perf_loads <= perf_loads + `PERF_CTR_BITS'(perf_dcache_rd_req_per_cycle); - perf_stores <= perf_stores + `PERF_CTR_BITS'(perf_dcache_wr_req_per_cycle); - perf_icache_lat <= perf_icache_lat + perf_icache_pending_reads; - perf_dcache_lat <= perf_dcache_lat + perf_dcache_pending_reads; - end - end - - assign pipeline_perf_if.ifetches = perf_ifetches; - assign pipeline_perf_if.loads = perf_loads; - assign pipeline_perf_if.stores = perf_stores; - assign pipeline_perf_if.load_latency = perf_dcache_lat; - assign pipeline_perf_if.ifetch_latency = perf_icache_lat; - assign pipeline_perf_if.load_latency = perf_dcache_lat; - -`endif - -endmodule +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef EXT_F_ENABLE +`include "VX_fpu_define.vh" +`endif + +module VX_core import VX_gpu_pkg::*; #( + parameter CORE_ID = 0 +) ( + `SCOPE_IO_DECL + + // Clock + input wire clk, + input wire reset, + +`ifdef PERF_ENABLE + VX_mem_perf_if.slave mem_perf_if, +`endif + + VX_dcr_bus_if.slave dcr_bus_if, + + VX_mem_bus_if.master smem_bus_if [DCACHE_NUM_REQS], + + VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], + + VX_mem_bus_if.master icache_bus_if, + +`ifdef GBAR_ENABLE + VX_gbar_bus_if.master gbar_bus_if, +`endif + + // simulation helper signals + output wire sim_ebreak, + output wire [`NUM_REGS-1:0][`XLEN-1:0] sim_wb_value, + + // Status + output wire busy +); + VX_schedule_if schedule_if(); + VX_fetch_if fetch_if(); + VX_decode_if decode_if(); + VX_sched_csr_if sched_csr_if(); + VX_decode_sched_if decode_sched_if(); + VX_commit_sched_if commit_sched_if(); + VX_commit_csr_if commit_csr_if(); + VX_branch_ctl_if branch_ctl_if[`NUM_ALU_BLOCKS](); + VX_warp_ctl_if warp_ctl_if(); + + VX_dispatch_if alu_dispatch_if[`ISSUE_WIDTH](); + VX_commit_if alu_commit_if[`ISSUE_WIDTH](); + + VX_dispatch_if lsu_dispatch_if[`ISSUE_WIDTH](); + VX_commit_if lsu_commit_if[`ISSUE_WIDTH](); +`ifdef EXT_F_ENABLE + VX_dispatch_if fpu_dispatch_if[`ISSUE_WIDTH](); + VX_commit_if fpu_commit_if[`ISSUE_WIDTH](); +`endif + VX_dispatch_if sfu_dispatch_if[`ISSUE_WIDTH](); + VX_commit_if sfu_commit_if[`ISSUE_WIDTH](); + + VX_writeback_if writeback_if[`ISSUE_WIDTH](); + + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) dcache_bus_tmp_if[DCACHE_NUM_REQS](); + +`ifdef PERF_ENABLE + VX_mem_perf_if mem_perf_tmp_if(); + VX_pipeline_perf_if pipeline_perf_if(); + + assign mem_perf_tmp_if.icache = mem_perf_if.icache; + assign mem_perf_tmp_if.dcache = mem_perf_if.dcache; + assign mem_perf_tmp_if.l2cache = mem_perf_if.l2cache; + assign mem_perf_tmp_if.l3cache = mem_perf_if.l3cache; + assign mem_perf_tmp_if.mem = mem_perf_if.mem; +`endif + + `RESET_RELAY (dcr_data_reset, reset); + `RESET_RELAY (schedule_reset, reset); + `RESET_RELAY (fetch_reset, reset); + `RESET_RELAY (decode_reset, reset); + `RESET_RELAY (issue_reset, reset); + `RESET_RELAY (execute_reset, reset); + `RESET_RELAY (commit_reset, reset); + + base_dcrs_t base_dcrs; + + VX_dcr_data dcr_data ( + .clk (clk), + .reset (dcr_data_reset), + .dcr_bus_if (dcr_bus_if), + .base_dcrs (base_dcrs) + ); + + `SCOPE_IO_SWITCH (3) + + VX_schedule #( + .CORE_ID (CORE_ID) + ) schedule ( + .clk (clk), + .reset (schedule_reset), + + `ifdef PERF_ENABLE + .perf_schedule_if (pipeline_perf_if.schedule), + `endif + + .base_dcrs (base_dcrs), + + .warp_ctl_if (warp_ctl_if), + .branch_ctl_if (branch_ctl_if), + .decode_sched_if(decode_sched_if), + .commit_sched_if(commit_sched_if), + + .schedule_if (schedule_if), + `ifdef GBAR_ENABLE + .gbar_bus_if (gbar_bus_if), + `endif + .sched_csr_if (sched_csr_if), + + .busy (busy) + ); + + VX_fetch #( + .CORE_ID (CORE_ID) + ) fetch ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (fetch_reset), + .icache_bus_if (icache_bus_if), + .schedule_if (schedule_if), + .fetch_if (fetch_if) + ); + + VX_decode #( + .CORE_ID (CORE_ID) + ) decode ( + .clk (clk), + .reset (decode_reset), + .fetch_if (fetch_if), + .decode_if (decode_if), + .decode_sched_if(decode_sched_if) + ); + + VX_issue #( + .CORE_ID (CORE_ID) + ) issue ( + `SCOPE_IO_BIND (1) + + .clk (clk), + .reset (issue_reset), + + `ifdef PERF_ENABLE + .perf_issue_if (pipeline_perf_if.issue), + `endif + + .decode_if (decode_if), + .writeback_if (writeback_if), + + .alu_dispatch_if(alu_dispatch_if), + .lsu_dispatch_if(lsu_dispatch_if), + `ifdef EXT_F_ENABLE + .fpu_dispatch_if(fpu_dispatch_if), + `endif + .sfu_dispatch_if(sfu_dispatch_if) + ); + + VX_execute #( + .CORE_ID (CORE_ID) + ) execute ( + `SCOPE_IO_BIND (2) + + .clk (clk), + .reset (execute_reset), + + .base_dcrs (base_dcrs), + + `ifdef PERF_ENABLE + .mem_perf_if (mem_perf_tmp_if), + .pipeline_perf_if(pipeline_perf_if), + `endif + + .dcache_bus_if (dcache_bus_tmp_if), + + `ifdef EXT_F_ENABLE + .fpu_dispatch_if(fpu_dispatch_if), + .fpu_commit_if (fpu_commit_if), + `endif + + .commit_csr_if (commit_csr_if), + .sched_csr_if (sched_csr_if), + + .alu_dispatch_if(alu_dispatch_if), + .lsu_dispatch_if(lsu_dispatch_if), + .sfu_dispatch_if(sfu_dispatch_if), + + .warp_ctl_if (warp_ctl_if), + .branch_ctl_if (branch_ctl_if), + + .alu_commit_if (alu_commit_if), + .lsu_commit_if (lsu_commit_if), + .sfu_commit_if (sfu_commit_if), + + .sim_ebreak (sim_ebreak) + ); + + VX_commit #( + .CORE_ID (CORE_ID) + ) commit ( + .clk (clk), + .reset (commit_reset), + + .alu_commit_if (alu_commit_if), + .lsu_commit_if (lsu_commit_if), + `ifdef EXT_F_ENABLE + .fpu_commit_if (fpu_commit_if), + `endif + .sfu_commit_if (sfu_commit_if), + + .writeback_if (writeback_if), + + .commit_csr_if (commit_csr_if), + .commit_sched_if(commit_sched_if), + + .sim_wb_value (sim_wb_value) + ); + +`ifdef SM_ENABLE + + VX_smem_unit #( + .CORE_ID (CORE_ID) + ) smem_unit ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + .cache_perf (mem_perf_tmp_if.smem), + `endif + .dcache_bus_in_if (dcache_bus_tmp_if), + .dcache_bus_out_if (dcache_bus_if), + .smem_bus_out_if (smem_bus_if) + ); + +`else + + for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin + `ASSIGN_VX_MEM_BUS_IF (dcache_bus_if[i], dcache_bus_tmp_if[i]); + end + +`endif + +`ifdef PERF_ENABLE + + wire [`CLOG2(DCACHE_NUM_REQS+1)-1:0] perf_dcache_rd_req_per_cycle; + wire [`CLOG2(DCACHE_NUM_REQS+1)-1:0] perf_dcache_wr_req_per_cycle; + wire [`CLOG2(DCACHE_NUM_REQS+1)-1:0] perf_dcache_rsp_per_cycle; + + wire [1:0] perf_icache_pending_read_cycle; + wire [`CLOG2(DCACHE_NUM_REQS+1)+1-1:0] perf_dcache_pending_read_cycle; + + reg [`PERF_CTR_BITS-1:0] perf_icache_pending_reads; + reg [`PERF_CTR_BITS-1:0] perf_dcache_pending_reads; + + reg [`PERF_CTR_BITS-1:0] perf_ifetches; + reg [`PERF_CTR_BITS-1:0] perf_loads; + reg [`PERF_CTR_BITS-1:0] perf_stores; + + wire perf_icache_req_fire = icache_bus_if.req_valid && icache_bus_if.req_ready; + wire perf_icache_rsp_fire = icache_bus_if.rsp_valid && icache_bus_if.rsp_ready; + + wire [DCACHE_NUM_REQS-1:0] perf_dcache_rd_req_fire, perf_dcache_rd_req_fire_r; + wire [DCACHE_NUM_REQS-1:0] perf_dcache_wr_req_fire, perf_dcache_wr_req_fire_r; + wire [DCACHE_NUM_REQS-1:0] perf_dcache_rsp_fire; + + for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin + assign perf_dcache_rd_req_fire[i] = dcache_bus_if[i].req_valid && dcache_bus_if[i].req_ready && ~dcache_bus_if[i].req_data.rw; + assign perf_dcache_wr_req_fire[i] = dcache_bus_if[i].req_valid && dcache_bus_if[i].req_ready && dcache_bus_if[i].req_data.rw; + assign perf_dcache_rsp_fire[i] = dcache_bus_if[i].rsp_valid && dcache_bus_if[i].rsp_ready; + end + + `BUFFER(perf_dcache_rd_req_fire_r, perf_dcache_rd_req_fire); + `BUFFER(perf_dcache_wr_req_fire_r, perf_dcache_wr_req_fire); + + `POP_COUNT(perf_dcache_rd_req_per_cycle, perf_dcache_rd_req_fire_r); + `POP_COUNT(perf_dcache_wr_req_per_cycle, perf_dcache_wr_req_fire_r); + `POP_COUNT(perf_dcache_rsp_per_cycle, perf_dcache_rsp_fire); + + assign perf_icache_pending_read_cycle = perf_icache_req_fire - perf_icache_rsp_fire; + assign perf_dcache_pending_read_cycle = perf_dcache_rd_req_per_cycle - perf_dcache_rsp_per_cycle; + + always @(posedge clk) begin + if (reset) begin + perf_icache_pending_reads <= '0; + perf_dcache_pending_reads <= '0; + end else begin + perf_icache_pending_reads <= $signed(perf_icache_pending_reads) + `PERF_CTR_BITS'($signed(perf_icache_pending_read_cycle)); + perf_dcache_pending_reads <= $signed(perf_dcache_pending_reads) + `PERF_CTR_BITS'($signed(perf_dcache_pending_read_cycle)); + end + end + + reg [`PERF_CTR_BITS-1:0] perf_icache_lat; + reg [`PERF_CTR_BITS-1:0] perf_dcache_lat; + + always @(posedge clk) begin + if (reset) begin + perf_ifetches <= '0; + perf_loads <= '0; + perf_stores <= '0; + perf_icache_lat <= '0; + perf_dcache_lat <= '0; + end else begin + perf_ifetches <= perf_ifetches + `PERF_CTR_BITS'(perf_icache_req_fire); + perf_loads <= perf_loads + `PERF_CTR_BITS'(perf_dcache_rd_req_per_cycle); + perf_stores <= perf_stores + `PERF_CTR_BITS'(perf_dcache_wr_req_per_cycle); + perf_icache_lat <= perf_icache_lat + perf_icache_pending_reads; + perf_dcache_lat <= perf_dcache_lat + perf_dcache_pending_reads; + end + end + + assign pipeline_perf_if.ifetches = perf_ifetches; + assign pipeline_perf_if.loads = perf_loads; + assign pipeline_perf_if.stores = perf_stores; + assign pipeline_perf_if.load_latency = perf_dcache_lat; + assign pipeline_perf_if.ifetch_latency = perf_icache_lat; + assign pipeline_perf_if.load_latency = perf_dcache_lat; + +`endif + +endmodule diff --git a/hw/rtl/core/VX_core_top.sv b/hw/rtl/core/VX_core_top.sv index 8d126f96..83318086 100644 --- a/hw/rtl/core/VX_core_top.sv +++ b/hw/rtl/core/VX_core_top.sv @@ -129,7 +129,13 @@ module VX_core_top import VX_gpu_pkg::*; #( assign icache_rsp_ready = icache_bus_if.rsp_ready; `ifdef PERF_ENABLE - VX_mem_perf_if mem_perf_if(); + VX_mem_perf_if mem_perf_if(); + assign mem_perf_if.icache = '0; + assign mem_perf_if.dcache = '0; + assign mem_perf_if.l2cache = '0; + assign mem_perf_if.l3cache = '0; + assign mem_perf_if.smem = '0; + assign mem_perf_if.mem = '0; `endif `ifdef SCOPE diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index 881d489e..10cc5f55 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -35,7 +35,6 @@ import VX_fpu_pkg::*; `ifdef PERF_ENABLE VX_mem_perf_if.slave mem_perf_if, VX_pipeline_perf_if.slave pipeline_perf_if, - VX_sfu_perf_if.slave sfu_perf_if, `endif VX_commit_csr_if.slave commit_csr_if, @@ -183,105 +182,115 @@ import VX_fpu_pkg::*; default: begin read_addr_valid_r = 0; - if ((read_addr >= `VX_CSR_MPM_BASE && read_addr < (`VX_CSR_MPM_BASE + 32)) - || (read_addr >= `VX_CSR_MPM_BASE_H && read_addr < (`VX_CSR_MPM_BASE_H + 32))) begin + if ((read_addr >= `VX_CSR_MPM_USER && read_addr < (`VX_CSR_MPM_USER + 32)) + || (read_addr >= `VX_CSR_MPM_USER_H && read_addr < (`VX_CSR_MPM_USER_H + 32))) begin read_addr_valid_r = 1; `ifdef PERF_ENABLE case (base_dcrs.mpm_class) `VX_DCR_MPM_CLASS_CORE: begin case (read_addr) // PERF: pipeline - `VX_CSR_MPM_IBUF_ST : read_data_ro_r = pipeline_perf_if.ibf_stalls[31:0]; - `VX_CSR_MPM_IBUF_ST_H : read_data_ro_r = 32'(pipeline_perf_if.ibf_stalls[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_SCRB_ST : read_data_ro_r = pipeline_perf_if.scb_stalls[31:0]; - `VX_CSR_MPM_SCRB_ST_H : read_data_ro_r = 32'(pipeline_perf_if.scb_stalls[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_ALU_ST : read_data_ro_r = pipeline_perf_if.dsp_stalls[`EX_ALU][31:0]; - `VX_CSR_MPM_ALU_ST_H : read_data_ro_r = 32'(pipeline_perf_if.dsp_stalls[`EX_ALU][`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_LSU_ST : read_data_ro_r = pipeline_perf_if.dsp_stalls[`EX_LSU][31:0]; - `VX_CSR_MPM_LSU_ST_H : read_data_ro_r = 32'(pipeline_perf_if.dsp_stalls[`EX_LSU][`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SCHED_ID : read_data_ro_r = pipeline_perf_if.sched_idles[31:0]; + `VX_CSR_MPM_SCHED_ID_H : read_data_ro_r = 32'(pipeline_perf_if.sched_idles[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SCHED_ST : read_data_ro_r = pipeline_perf_if.sched_stalls[31:0]; + `VX_CSR_MPM_SCHED_ST_H : read_data_ro_r = 32'(pipeline_perf_if.sched_stalls[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_IBUF_ST : read_data_ro_r = pipeline_perf_if.ibf_stalls[31:0]; + `VX_CSR_MPM_IBUF_ST_H : read_data_ro_r = 32'(pipeline_perf_if.ibf_stalls[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SCRB_ST : read_data_ro_r = pipeline_perf_if.scb_stalls[31:0]; + `VX_CSR_MPM_SCRB_ST_H : read_data_ro_r = 32'(pipeline_perf_if.scb_stalls[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SCRB_ALU : read_data_ro_r = pipeline_perf_if.units_uses[`EX_ALU][31:0]; + `VX_CSR_MPM_SCRB_ALU_H : read_data_ro_r = 32'(pipeline_perf_if.units_uses[`EX_ALU][`PERF_CTR_BITS-1:32]); `ifdef EXT_F_ENABLE - `VX_CSR_MPM_FPU_ST : read_data_ro_r = pipeline_perf_if.dsp_stalls[`EX_FPU][31:0]; - `VX_CSR_MPM_FPU_ST_H : read_data_ro_r = 32'(pipeline_perf_if.dsp_stalls[`EX_FPU][`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SCRB_FPU : read_data_ro_r = pipeline_perf_if.units_uses[`EX_FPU][31:0]; + `VX_CSR_MPM_SCRB_FPU_H : read_data_ro_r = 32'(pipeline_perf_if.units_uses[`EX_FPU][`PERF_CTR_BITS-1:32]); `else - `VX_CSR_MPM_FPU_ST : read_data_ro_r = '0; - `VX_CSR_MPM_FPU_ST_H : read_data_ro_r = '0; + `VX_CSR_MPM_SCRB_FPU : read_data_ro_r = '0; + `VX_CSR_MPM_SCRB_FPU_H : read_data_ro_r = '0; `endif - `VX_CSR_MPM_SFU_ST : read_data_ro_r = pipeline_perf_if.dsp_stalls[`EX_SFU][31:0]; - `VX_CSR_MPM_SFU_ST_H : read_data_ro_r = 32'(pipeline_perf_if.dsp_stalls[`EX_SFU][`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SCRB_LSU : read_data_ro_r = pipeline_perf_if.units_uses[`EX_LSU][31:0]; + `VX_CSR_MPM_SCRB_LSU_H : read_data_ro_r = 32'(pipeline_perf_if.units_uses[`EX_LSU][`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SCRB_SFU : read_data_ro_r = pipeline_perf_if.units_uses[`EX_SFU][31:0]; + `VX_CSR_MPM_SCRB_SFU_H : read_data_ro_r = 32'(pipeline_perf_if.units_uses[`EX_SFU][`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SCRB_CSRS : read_data_ro_r = pipeline_perf_if.sfu_uses[`SFU_CSRS][31:0]; + `VX_CSR_MPM_SCRB_CSRS_H : read_data_ro_r = 32'(pipeline_perf_if.sfu_uses[`SFU_CSRS][`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SCRB_WCTL : read_data_ro_r = pipeline_perf_if.sfu_uses[`SFU_WCTL][31:0]; + `VX_CSR_MPM_SCRB_WCTL_H : read_data_ro_r = 32'(pipeline_perf_if.sfu_uses[`SFU_WCTL][`PERF_CTR_BITS-1:32]); // PERF: memory - `VX_CSR_MPM_IFETCHES : read_data_ro_r = pipeline_perf_if.ifetches[31:0]; - `VX_CSR_MPM_IFETCHES_H : read_data_ro_r = 32'(pipeline_perf_if.ifetches[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_LOADS : read_data_ro_r = pipeline_perf_if.loads[31:0]; - `VX_CSR_MPM_LOADS_H : read_data_ro_r = 32'(pipeline_perf_if.loads[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_STORES : read_data_ro_r = pipeline_perf_if.stores[31:0]; - `VX_CSR_MPM_STORES_H : read_data_ro_r = 32'(pipeline_perf_if.stores[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_IFETCH_LAT : read_data_ro_r = pipeline_perf_if.ifetch_latency[31:0]; - `VX_CSR_MPM_IFETCH_LAT_H : read_data_ro_r = 32'(pipeline_perf_if.ifetch_latency[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_LOAD_LAT : read_data_ro_r = pipeline_perf_if.load_latency[31:0]; - `VX_CSR_MPM_LOAD_LAT_H : read_data_ro_r = 32'(pipeline_perf_if.load_latency[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_IFETCHES : read_data_ro_r = pipeline_perf_if.ifetches[31:0]; + `VX_CSR_MPM_IFETCHES_H : read_data_ro_r = 32'(pipeline_perf_if.ifetches[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_LOADS : read_data_ro_r = pipeline_perf_if.loads[31:0]; + `VX_CSR_MPM_LOADS_H : read_data_ro_r = 32'(pipeline_perf_if.loads[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_STORES : read_data_ro_r = pipeline_perf_if.stores[31:0]; + `VX_CSR_MPM_STORES_H : read_data_ro_r = 32'(pipeline_perf_if.stores[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_IFETCH_LT : read_data_ro_r = pipeline_perf_if.ifetch_latency[31:0]; + `VX_CSR_MPM_IFETCH_LT_H : read_data_ro_r = 32'(pipeline_perf_if.ifetch_latency[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_LOAD_LT : read_data_ro_r = pipeline_perf_if.load_latency[31:0]; + `VX_CSR_MPM_LOAD_LT_H : read_data_ro_r = 32'(pipeline_perf_if.load_latency[`PERF_CTR_BITS-1:32]); default:; endcase end `VX_DCR_MPM_CLASS_MEM: begin case (read_addr) // PERF: icache - `VX_CSR_MPM_ICACHE_READS : read_data_ro_r = mem_perf_if.icache.reads[31:0]; - `VX_CSR_MPM_ICACHE_READS_H : read_data_ro_r = 32'(mem_perf_if.icache.reads[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_ICACHE_MISS_R : read_data_ro_r = mem_perf_if.icache.read_misses[31:0]; - `VX_CSR_MPM_ICACHE_MISS_R_H : read_data_ro_r = 32'(mem_perf_if.icache.read_misses[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_ICACHE_READS : read_data_ro_r = mem_perf_if.icache.reads[31:0]; + `VX_CSR_MPM_ICACHE_READS_H : read_data_ro_r = 32'(mem_perf_if.icache.reads[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_ICACHE_MISS_R : read_data_ro_r = mem_perf_if.icache.read_misses[31:0]; + `VX_CSR_MPM_ICACHE_MISS_R_H : read_data_ro_r = 32'(mem_perf_if.icache.read_misses[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_ICACHE_MSHR_ST : read_data_ro_r = mem_perf_if.icache.mshr_stalls[31:0]; + `VX_CSR_MPM_ICACHE_MSHR_ST_H: read_data_ro_r = 32'(mem_perf_if.icache.mshr_stalls[`PERF_CTR_BITS-1:32]); // PERF: dcache - `VX_CSR_MPM_DCACHE_READS : read_data_ro_r = mem_perf_if.dcache.reads[31:0]; - `VX_CSR_MPM_DCACHE_READS_H : read_data_ro_r = 32'(mem_perf_if.dcache.reads[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_DCACHE_WRITES : read_data_ro_r = mem_perf_if.dcache.writes[31:0]; - `VX_CSR_MPM_DCACHE_WRITES_H : read_data_ro_r = 32'(mem_perf_if.dcache.writes[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_DCACHE_MISS_R : read_data_ro_r = mem_perf_if.dcache.read_misses[31:0]; - `VX_CSR_MPM_DCACHE_MISS_R_H : read_data_ro_r = 32'(mem_perf_if.dcache.read_misses[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_DCACHE_MISS_W : read_data_ro_r = mem_perf_if.dcache.write_misses[31:0]; - `VX_CSR_MPM_DCACHE_MISS_W_H : read_data_ro_r = 32'(mem_perf_if.dcache.write_misses[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_DCACHE_BANK_ST : read_data_ro_r = mem_perf_if.dcache.bank_stalls[31:0]; - `VX_CSR_MPM_DCACHE_BANK_ST_H : read_data_ro_r = 32'(mem_perf_if.dcache.bank_stalls[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_DCACHE_MSHR_ST : read_data_ro_r = mem_perf_if.dcache.mshr_stalls[31:0]; - `VX_CSR_MPM_DCACHE_MSHR_ST_H : read_data_ro_r = 32'(mem_perf_if.dcache.mshr_stalls[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_DCACHE_READS : read_data_ro_r = mem_perf_if.dcache.reads[31:0]; + `VX_CSR_MPM_DCACHE_READS_H : read_data_ro_r = 32'(mem_perf_if.dcache.reads[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_DCACHE_WRITES : read_data_ro_r = mem_perf_if.dcache.writes[31:0]; + `VX_CSR_MPM_DCACHE_WRITES_H : read_data_ro_r = 32'(mem_perf_if.dcache.writes[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_DCACHE_MISS_R : read_data_ro_r = mem_perf_if.dcache.read_misses[31:0]; + `VX_CSR_MPM_DCACHE_MISS_R_H : read_data_ro_r = 32'(mem_perf_if.dcache.read_misses[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_DCACHE_MISS_W : read_data_ro_r = mem_perf_if.dcache.write_misses[31:0]; + `VX_CSR_MPM_DCACHE_MISS_W_H : read_data_ro_r = 32'(mem_perf_if.dcache.write_misses[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_DCACHE_BANK_ST : read_data_ro_r = mem_perf_if.dcache.bank_stalls[31:0]; + `VX_CSR_MPM_DCACHE_BANK_ST_H: read_data_ro_r = 32'(mem_perf_if.dcache.bank_stalls[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_DCACHE_MSHR_ST : read_data_ro_r = mem_perf_if.dcache.mshr_stalls[31:0]; + `VX_CSR_MPM_DCACHE_MSHR_ST_H: read_data_ro_r = 32'(mem_perf_if.dcache.mshr_stalls[`PERF_CTR_BITS-1:32]); // PERF: smem - `VX_CSR_MPM_SMEM_READS : read_data_ro_r = mem_perf_if.smem.reads[31:0]; - `VX_CSR_MPM_SMEM_READS_H : read_data_ro_r = 32'(mem_perf_if.smem.reads[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_SMEM_WRITES : read_data_ro_r = mem_perf_if.smem.writes[31:0]; - `VX_CSR_MPM_SMEM_WRITES_H : read_data_ro_r = 32'(mem_perf_if.smem.writes[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_SMEM_BANK_ST : read_data_ro_r = mem_perf_if.smem.bank_stalls[31:0]; - `VX_CSR_MPM_SMEM_BANK_ST_H : read_data_ro_r = 32'(mem_perf_if.smem.bank_stalls[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SMEM_READS : read_data_ro_r = mem_perf_if.smem.reads[31:0]; + `VX_CSR_MPM_SMEM_READS_H : read_data_ro_r = 32'(mem_perf_if.smem.reads[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SMEM_WRITES : read_data_ro_r = mem_perf_if.smem.writes[31:0]; + `VX_CSR_MPM_SMEM_WRITES_H : read_data_ro_r = 32'(mem_perf_if.smem.writes[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_SMEM_BANK_ST : read_data_ro_r = mem_perf_if.smem.bank_stalls[31:0]; + `VX_CSR_MPM_SMEM_BANK_ST_H : read_data_ro_r = 32'(mem_perf_if.smem.bank_stalls[`PERF_CTR_BITS-1:32]); // PERF: l2cache - `VX_CSR_MPM_L2CACHE_READS : read_data_ro_r = mem_perf_if.l2cache.reads[31:0]; - `VX_CSR_MPM_L2CACHE_READS_H : read_data_ro_r = 32'(mem_perf_if.l2cache.reads[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_L2CACHE_WRITES : read_data_ro_r = mem_perf_if.l2cache.writes[31:0]; - `VX_CSR_MPM_L2CACHE_WRITES_H : read_data_ro_r = 32'(mem_perf_if.l2cache.writes[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_L2CACHE_MISS_R : read_data_ro_r = mem_perf_if.l2cache.read_misses[31:0]; - `VX_CSR_MPM_L2CACHE_MISS_R_H : read_data_ro_r = 32'(mem_perf_if.l2cache.read_misses[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_L2CACHE_MISS_W : read_data_ro_r = mem_perf_if.l2cache.write_misses[31:0]; - `VX_CSR_MPM_L2CACHE_MISS_W_H : read_data_ro_r = 32'(mem_perf_if.l2cache.write_misses[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_L2CACHE_BANK_ST : read_data_ro_r = mem_perf_if.l2cache.bank_stalls[31:0]; - `VX_CSR_MPM_L2CACHE_BANK_ST_H : read_data_ro_r = 32'(mem_perf_if.l2cache.bank_stalls[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_L2CACHE_MSHR_ST : read_data_ro_r = mem_perf_if.l2cache.mshr_stalls[31:0]; - `VX_CSR_MPM_L2CACHE_MSHR_ST_H : read_data_ro_r = 32'(mem_perf_if.l2cache.mshr_stalls[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L2CACHE_READS : read_data_ro_r = mem_perf_if.l2cache.reads[31:0]; + `VX_CSR_MPM_L2CACHE_READS_H : read_data_ro_r = 32'(mem_perf_if.l2cache.reads[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L2CACHE_WRITES : read_data_ro_r = mem_perf_if.l2cache.writes[31:0]; + `VX_CSR_MPM_L2CACHE_WRITES_H: read_data_ro_r = 32'(mem_perf_if.l2cache.writes[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L2CACHE_MISS_R : read_data_ro_r = mem_perf_if.l2cache.read_misses[31:0]; + `VX_CSR_MPM_L2CACHE_MISS_R_H: read_data_ro_r = 32'(mem_perf_if.l2cache.read_misses[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L2CACHE_MISS_W : read_data_ro_r = mem_perf_if.l2cache.write_misses[31:0]; + `VX_CSR_MPM_L2CACHE_MISS_W_H: read_data_ro_r = 32'(mem_perf_if.l2cache.write_misses[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L2CACHE_BANK_ST : read_data_ro_r = mem_perf_if.l2cache.bank_stalls[31:0]; + `VX_CSR_MPM_L2CACHE_BANK_ST_H: read_data_ro_r = 32'(mem_perf_if.l2cache.bank_stalls[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L2CACHE_MSHR_ST : read_data_ro_r = mem_perf_if.l2cache.mshr_stalls[31:0]; + `VX_CSR_MPM_L2CACHE_MSHR_ST_H: read_data_ro_r = 32'(mem_perf_if.l2cache.mshr_stalls[`PERF_CTR_BITS-1:32]); // PERF: l3cache - `VX_CSR_MPM_L3CACHE_READS : read_data_ro_r = mem_perf_if.l3cache.reads[31:0]; - `VX_CSR_MPM_L3CACHE_READS_H : read_data_ro_r = 32'(mem_perf_if.l3cache.reads[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_L3CACHE_WRITES : read_data_ro_r = mem_perf_if.l3cache.writes[31:0]; - `VX_CSR_MPM_L3CACHE_WRITES_H : read_data_ro_r = 32'(mem_perf_if.l3cache.writes[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_L3CACHE_MISS_R : read_data_ro_r = mem_perf_if.l3cache.read_misses[31:0]; - `VX_CSR_MPM_L3CACHE_MISS_R_H : read_data_ro_r = 32'(mem_perf_if.l3cache.read_misses[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_L3CACHE_MISS_W : read_data_ro_r = mem_perf_if.l3cache.write_misses[31:0]; - `VX_CSR_MPM_L3CACHE_MISS_W_H : read_data_ro_r = 32'(mem_perf_if.l3cache.write_misses[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_L3CACHE_BANK_ST : read_data_ro_r = mem_perf_if.l3cache.bank_stalls[31:0]; - `VX_CSR_MPM_L3CACHE_BANK_ST_H : read_data_ro_r = 32'(mem_perf_if.l3cache.bank_stalls[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_L3CACHE_MSHR_ST : read_data_ro_r = mem_perf_if.l3cache.mshr_stalls[31:0]; - `VX_CSR_MPM_L3CACHE_MSHR_ST_H : read_data_ro_r = 32'(mem_perf_if.l3cache.mshr_stalls[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L3CACHE_READS : read_data_ro_r = mem_perf_if.l3cache.reads[31:0]; + `VX_CSR_MPM_L3CACHE_READS_H : read_data_ro_r = 32'(mem_perf_if.l3cache.reads[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L3CACHE_WRITES : read_data_ro_r = mem_perf_if.l3cache.writes[31:0]; + `VX_CSR_MPM_L3CACHE_WRITES_H: read_data_ro_r = 32'(mem_perf_if.l3cache.writes[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L3CACHE_MISS_R : read_data_ro_r = mem_perf_if.l3cache.read_misses[31:0]; + `VX_CSR_MPM_L3CACHE_MISS_R_H: read_data_ro_r = 32'(mem_perf_if.l3cache.read_misses[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L3CACHE_MISS_W : read_data_ro_r = mem_perf_if.l3cache.write_misses[31:0]; + `VX_CSR_MPM_L3CACHE_MISS_W_H: read_data_ro_r = 32'(mem_perf_if.l3cache.write_misses[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L3CACHE_BANK_ST : read_data_ro_r = mem_perf_if.l3cache.bank_stalls[31:0]; + `VX_CSR_MPM_L3CACHE_BANK_ST_H: read_data_ro_r = 32'(mem_perf_if.l3cache.bank_stalls[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_L3CACHE_MSHR_ST : read_data_ro_r = mem_perf_if.l3cache.mshr_stalls[31:0]; + `VX_CSR_MPM_L3CACHE_MSHR_ST_H: read_data_ro_r = 32'(mem_perf_if.l3cache.mshr_stalls[`PERF_CTR_BITS-1:32]); // PERF: memory - `VX_CSR_MPM_MEM_READS : read_data_ro_r = mem_perf_if.mem.reads[31:0]; - `VX_CSR_MPM_MEM_READS_H : read_data_ro_r = 32'(mem_perf_if.mem.reads[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_MEM_WRITES : read_data_ro_r = mem_perf_if.mem.writes[31:0]; - `VX_CSR_MPM_MEM_WRITES_H : read_data_ro_r = 32'(mem_perf_if.mem.writes[`PERF_CTR_BITS-1:32]); - `VX_CSR_MPM_MEM_LAT : read_data_ro_r = mem_perf_if.mem.latency[31:0]; - `VX_CSR_MPM_MEM_LAT_H : read_data_ro_r = 32'(mem_perf_if.mem.latency[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_MEM_READS : read_data_ro_r = mem_perf_if.mem.reads[31:0]; + `VX_CSR_MPM_MEM_READS_H : read_data_ro_r = 32'(mem_perf_if.mem.reads[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_MEM_WRITES : read_data_ro_r = mem_perf_if.mem.writes[31:0]; + `VX_CSR_MPM_MEM_WRITES_H : read_data_ro_r = 32'(mem_perf_if.mem.writes[`PERF_CTR_BITS-1:32]); + `VX_CSR_MPM_MEM_LT : read_data_ro_r = mem_perf_if.mem.latency[31:0]; + `VX_CSR_MPM_MEM_LT_H : read_data_ro_r = 32'(mem_perf_if.mem.latency[`PERF_CTR_BITS-1:32]); default:; endcase end @@ -301,8 +310,6 @@ import VX_fpu_pkg::*; `RUNTIME_ASSERT(~read_enable || read_addr_valid_r, ("%t: *** invalid CSR read address: 0x%0h (#%0d)", $time, read_addr, read_uuid)) `ifdef PERF_ENABLE - wire [`PERF_CTR_BITS-1:0] perf_wctl_stalls = sfu_perf_if.wctl_stalls; - `UNUSED_VAR (perf_wctl_stalls); `UNUSED_VAR (mem_perf_if.icache); `UNUSED_VAR (mem_perf_if.smem); `endif diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index 14b633fa..91cb37ab 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -25,7 +25,6 @@ module VX_csr_unit import VX_gpu_pkg::*; #( `ifdef PERF_ENABLE VX_mem_perf_if.slave mem_perf_if, VX_pipeline_perf_if.slave pipeline_perf_if, - VX_sfu_perf_if.slave sfu_perf_if, `endif `ifdef EXT_F_ENABLE @@ -81,7 +80,6 @@ module VX_csr_unit import VX_gpu_pkg::*; #( `ifdef PERF_ENABLE .mem_perf_if (mem_perf_if), .pipeline_perf_if(pipeline_perf_if), - .sfu_perf_if (sfu_perf_if), `endif .commit_csr_if (commit_csr_if), diff --git a/hw/rtl/core/VX_dispatch.sv b/hw/rtl/core/VX_dispatch.sv index f10caa47..7ae67607 100644 --- a/hw/rtl/core/VX_dispatch.sv +++ b/hw/rtl/core/VX_dispatch.sv @@ -12,6 +12,7 @@ // limitations under the License. `include "VX_define.vh" +`include "VX_trace.vh" module VX_dispatch import VX_gpu_pkg::*; #( parameter CORE_ID = 0 @@ -174,30 +175,38 @@ module VX_dispatch import VX_gpu_pkg::*; #( || (sfu_operands_if[i].ready && (operands_if[i].data.ex_type == `EX_SFU)); end -`ifdef PERF_ENABLE - reg [`NUM_EX_UNITS-1:0][`PERF_CTR_BITS-1:0] perf_stalls_n, perf_stalls_r; - wire [`ISSUE_WIDTH-1:0] operands_stall; - wire [`ISSUE_WIDTH-1:0][`EX_BITS-1:0] operands_ex_type; +`ifdef PERF_ENABLE + wire [`NUM_EX_UNITS-1:0] perf_unit_stalls_per_cycle, perf_unit_stalls_per_cycle_r; + reg [`ISSUE_WIDTH-1:0][`NUM_EX_UNITS-1:0] perf_issue_unit_stalls_per_cycle; + reg [`NUM_EX_UNITS-1:0][`PERF_CTR_BITS-1:0] perf_stalls_r; for (genvar i=0; i < `ISSUE_WIDTH; ++i) begin - assign operands_stall[i] = operands_if[i].valid && ~operands_if[i].ready; - assign operands_ex_type[i] = operands_if[i].data.ex_type; - end - - always @(*) begin - perf_stalls_n = perf_stalls_r; - for (integer i=0; i < `ISSUE_WIDTH; ++i) begin - if (operands_stall[i]) begin - perf_stalls_n[operands_ex_type[i]] += `PERF_CTR_BITS'(1); + always @(*) begin + perf_issue_unit_stalls_per_cycle[i] = '0; + if (operands_if[i].valid && ~operands_if[i].ready) begin + perf_issue_unit_stalls_per_cycle[i][operands_if[i].data.ex_type] = 1; end end end - always @(posedge clk) begin - if (reset) begin - perf_stalls_r <= '0; - end else begin - perf_stalls_r <= perf_stalls_n; + VX_reduce #( + .DATAW_IN (`NUM_EX_UNITS), + .N (`ISSUE_WIDTH), + .OP ("|") + ) reduce ( + .data_in (perf_issue_unit_stalls_per_cycle), + .data_out (perf_unit_stalls_per_cycle) + ); + + `BUFFER(perf_unit_stalls_per_cycle_r, perf_unit_stalls_per_cycle); + + for (genvar i = 0; i < `NUM_EX_UNITS; ++i) begin + always @(posedge clk) begin + if (reset) begin + perf_stalls_r[i] <= '0; + end else begin + perf_stalls_r[i] <= perf_stalls_r[i] + `PERF_CTR_BITS'(perf_unit_stalls_per_cycle_r[i]); + end end end diff --git a/hw/rtl/core/VX_dispatch_unit.sv b/hw/rtl/core/VX_dispatch_unit.sv index 86564187..6e36a33b 100644 --- a/hw/rtl/core/VX_dispatch_unit.sv +++ b/hw/rtl/core/VX_dispatch_unit.sv @@ -70,8 +70,8 @@ module VX_dispatch_unit import VX_gpu_pkg::*; #( always @(posedge clk) begin if (reset) begin batch_idx <= '0; - end else if (batch_done) begin - batch_idx <= batch_idx + BATCH_COUNT_W'(1); + end else begin + batch_idx <= batch_idx + BATCH_COUNT_W'(batch_done); end end end else begin @@ -203,20 +203,20 @@ module VX_dispatch_unit import VX_gpu_pkg::*; #( assign block_done[block_idx] = ~valid_p || ready_p; end - wire [ISSUE_IDX_W-1:0] wsi; + wire [ISSUE_ISW_W-1:0] isw; if (BATCH_COUNT != 1) begin if (BLOCK_SIZE != 1) begin - assign wsi = {batch_idx, BLOCK_SIZE_W'(block_idx)}; + assign isw = {batch_idx, BLOCK_SIZE_W'(block_idx)}; end else begin - assign wsi = batch_idx; + assign isw = batch_idx; end end else begin - assign wsi = block_idx; + assign isw = block_idx; end `RESET_RELAY(buf_out_reset, reset); - wire [`NW_WIDTH-1:0] block_wid = wis_to_wid(dispatch_data[issue_idx][DATA_TMASK_OFF+`NUM_THREADS +: ISSUE_WIS_W], wsi); + wire [`NW_WIDTH-1:0] block_wid = wis_to_wid(dispatch_data[issue_idx][DATA_TMASK_OFF+`NUM_THREADS +: ISSUE_WIS_W], isw); VX_elastic_buffer #( .DATAW (OUT_DATAW), diff --git a/hw/rtl/fpu/VX_fpu_unit.sv b/hw/rtl/core/VX_fpu_unit.sv similarity index 99% rename from hw/rtl/fpu/VX_fpu_unit.sv rename to hw/rtl/core/VX_fpu_unit.sv index 1aecd3e8..e00883b5 100644 --- a/hw/rtl/fpu/VX_fpu_unit.sv +++ b/hw/rtl/core/VX_fpu_unit.sv @@ -30,7 +30,7 @@ module VX_fpu_unit import VX_fpu_pkg::*; #( localparam NUM_LANES = `NUM_FPU_LANES; localparam PID_BITS = `CLOG2(`NUM_THREADS / NUM_LANES); localparam PID_WIDTH = `UP(PID_BITS); - localparam TAG_WIDTH = `LOG2UP(`FPU_REQ_QUEUE_SIZE); + localparam TAG_WIDTH = `LOG2UP(`FPUQ_SIZE); localparam PARTIAL_BW = (BLOCK_SIZE != `ISSUE_WIDTH) || (NUM_LANES != `NUM_THREADS); VX_execute_if #( @@ -87,7 +87,7 @@ module VX_fpu_unit import VX_fpu_pkg::*; #( VX_index_buffer #( .DATAW (`UUID_WIDTH + `NW_WIDTH + NUM_LANES + `XLEN + `NR_BITS + PID_WIDTH + 1 + 1), - .SIZE (`FPU_REQ_QUEUE_SIZE) + .SIZE (`FPUQ_SIZE) ) tag_store ( .clk (clk), .reset (reset), diff --git a/hw/rtl/core/VX_gather_unit.sv b/hw/rtl/core/VX_gather_unit.sv index e3dc935d..21ae4485 100644 --- a/hw/rtl/core/VX_gather_unit.sv +++ b/hw/rtl/core/VX_gather_unit.sv @@ -37,7 +37,7 @@ module VX_gather_unit import VX_gpu_pkg::*; #( wire [BLOCK_SIZE-1:0] commit_in_valid; wire [BLOCK_SIZE-1:0][DATAW-1:0] commit_in_data; wire [BLOCK_SIZE-1:0] commit_in_ready; - wire [BLOCK_SIZE-1:0][ISSUE_IDX_W-1:0] commit_in_wsi; + wire [BLOCK_SIZE-1:0][ISSUE_ISW_W-1:0] commit_in_isw; for (genvar i = 0; i < BLOCK_SIZE; ++i) begin assign commit_in_valid[i] = commit_in_if[i].valid; @@ -45,12 +45,12 @@ module VX_gather_unit import VX_gpu_pkg::*; #( assign commit_in_if[i].ready = commit_in_ready[i]; if (BLOCK_SIZE != `ISSUE_WIDTH) begin if (BLOCK_SIZE != 1) begin - assign commit_in_wsi[i] = {commit_in_data[i][DATA_WIS_OFF+BLOCK_SIZE_W +: (ISSUE_IDX_W-BLOCK_SIZE_W)], BLOCK_SIZE_W'(i)}; + assign commit_in_isw[i] = {commit_in_data[i][DATA_WIS_OFF+BLOCK_SIZE_W +: (ISSUE_ISW_W-BLOCK_SIZE_W)], BLOCK_SIZE_W'(i)}; end else begin - assign commit_in_wsi[i] = commit_in_data[i][DATA_WIS_OFF +: ISSUE_IDX_W]; + assign commit_in_isw[i] = commit_in_data[i][DATA_WIS_OFF +: ISSUE_ISW_W]; end end else begin - assign commit_in_wsi[i] = BLOCK_SIZE_W'(i); + assign commit_in_isw[i] = BLOCK_SIZE_W'(i); end end @@ -64,12 +64,12 @@ module VX_gather_unit import VX_gpu_pkg::*; #( commit_out_data[i] = 'x; end for (integer i = 0; i < BLOCK_SIZE; ++i) begin - commit_out_valid[commit_in_wsi[i]] = commit_in_valid[i]; - commit_out_data[commit_in_wsi[i]] = commit_in_data[i]; + commit_out_valid[commit_in_isw[i]] = commit_in_valid[i]; + commit_out_data[commit_in_isw[i]] = commit_in_data[i]; end end for (genvar i = 0; i < BLOCK_SIZE; ++i) begin - assign commit_in_ready[i] = commit_out_ready[commit_in_wsi[i]]; + assign commit_in_ready[i] = commit_out_ready[commit_in_isw[i]]; end for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin diff --git a/hw/rtl/core/VX_ipdom_stack.sv b/hw/rtl/core/VX_ipdom_stack.sv index a6524b2d..b6763f7e 100644 --- a/hw/rtl/core/VX_ipdom_stack.sv +++ b/hw/rtl/core/VX_ipdom_stack.sv @@ -14,10 +14,10 @@ `include "VX_platform.vh" module VX_ipdom_stack #( - parameter WIDTH = 1, - parameter DEPTH = 1, + parameter WIDTH = 1, + parameter DEPTH = 1, parameter OUT_REG = 0, - parameter ADDRW = `LOG2UP(DEPTH) + parameter ADDRW = `LOG2UP(DEPTH) ) ( input wire clk, input wire reset, diff --git a/hw/rtl/core/VX_issue.sv b/hw/rtl/core/VX_issue.sv index af00014e..1ba4ca28 100644 --- a/hw/rtl/core/VX_issue.sv +++ b/hw/rtl/core/VX_issue.sv @@ -59,6 +59,11 @@ module VX_issue #( ) scoreboard ( .clk (clk), .reset (scoreboard_reset), + `ifdef PERF_ENABLE + .perf_scb_stalls(perf_issue_if.scb_stalls), + .perf_units_uses(perf_issue_if.units_uses), + .perf_sfu_uses (perf_issue_if.sfu_uses), + `endif .writeback_if (writeback_if), .ibuffer_if (ibuffer_if), .scoreboard_if (scoreboard_if) @@ -80,7 +85,7 @@ module VX_issue #( .clk (clk), .reset (dispatch_reset), `ifdef PERF_ENABLE - .perf_stalls (perf_issue_if.dsp_stalls), + `UNUSED_PIN (perf_stalls), `endif .operands_if (operands_if), .alu_dispatch_if(alu_dispatch_if), @@ -152,29 +157,18 @@ module VX_issue #( `ifdef PERF_ENABLE reg [`PERF_CTR_BITS-1:0] perf_ibf_stalls; - reg [`PERF_CTR_BITS-1:0] perf_scb_stalls; - - wire [`CLOG2(`ISSUE_WIDTH+1)-1:0] scoreboard_stalls_per_cycle; - reg [`ISSUE_WIDTH-1:0] scoreboard_stalls; - for (genvar i=0; i < `ISSUE_WIDTH; ++i) begin - assign scoreboard_stalls[i] = ibuffer_if[i].valid && ~ibuffer_if[i].ready; - end - `POP_COUNT(scoreboard_stalls_per_cycle, scoreboard_stalls); + + wire decode_stall = decode_if.valid && ~decode_if.ready; always @(posedge clk) begin if (reset) begin perf_ibf_stalls <= '0; - perf_scb_stalls <= '0; end else begin - if (decode_if.valid && ~decode_if.ready) begin - perf_ibf_stalls <= perf_ibf_stalls + `PERF_CTR_BITS'(1); - end - perf_scb_stalls <= perf_scb_stalls + `PERF_CTR_BITS'(scoreboard_stalls_per_cycle); + perf_ibf_stalls <= perf_ibf_stalls + `PERF_CTR_BITS'(decode_stall); end end assign perf_issue_if.ibf_stalls = perf_ibf_stalls; - assign perf_issue_if.scb_stalls = perf_scb_stalls; `endif endmodule diff --git a/hw/rtl/core/VX_lsu_unit.sv b/hw/rtl/core/VX_lsu_unit.sv index 36aa76d5..d823982c 100644 --- a/hw/rtl/core/VX_lsu_unit.sv +++ b/hw/rtl/core/VX_lsu_unit.sv @@ -96,7 +96,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( // detect duplicate addresses wire lsu_is_dup; -`ifdef LSU_DUP +`ifdef LSU_DUP_ENABLE if (NUM_LANES > 1) begin wire [NUM_LANES-2:0] addr_matches; for (genvar i = 0; i < (NUM_LANES-1); ++i) begin @@ -304,7 +304,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( assign mem_req_tag = { execute_if[0].data.uuid, lsu_addr_type, execute_if[0].data.wid, execute_if[0].data.tmask, execute_if[0].data.PC, execute_if[0].data.rd, execute_if[0].data.op_type, req_align, execute_if[0].data.pid, pkt_waddr - `ifdef LSU_DUP + `ifdef LSU_DUP_ENABLE , lsu_is_dup `endif }; @@ -448,13 +448,13 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( wire [PID_WIDTH-1:0] rsp_pid; wire rsp_is_dup; -`ifndef LSU_DUP +`ifndef LSU_DUP_ENABLE assign rsp_is_dup = 0; `endif assign { rsp_uuid, rsp_addr_type, rsp_wid, rsp_tmask_uq, rsp_pc, rsp_rd, rsp_op_type, rsp_align, rsp_pid, pkt_raddr - `ifdef LSU_DUP + `ifdef LSU_DUP_ENABLE , rsp_is_dup `endif } = mem_rsp_tag; @@ -554,7 +554,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( VX_stream_arb #( .NUM_INPUTS (2), .DATAW (RSP_ARB_DATAW), - .OUT_REG (1) + .OUT_REG (3) ) rsp_arb ( .clk (clk), .reset (commit_reset), diff --git a/hw/rtl/core/VX_muldiv_unit.sv b/hw/rtl/core/VX_muldiv_unit.sv index 213b6677..6daa3c3d 100644 --- a/hw/rtl/core/VX_muldiv_unit.sv +++ b/hw/rtl/core/VX_muldiv_unit.sv @@ -220,8 +220,13 @@ module VX_muldiv_unit #( wire [NUM_LANES-1:0][`XLEN-1:0] div_in2; for (genvar i = 0; i < NUM_LANES; ++i) begin + `ifdef XLEN_64 assign div_in1[i] = is_alu_w ? {{(`XLEN-32){is_signed_op && execute_if.data.rs1_data[i][31]}}, execute_if.data.rs1_data[i][31:0]}: execute_if.data.rs1_data[i]; assign div_in2[i] = is_alu_w ? {{(`XLEN-32){is_signed_op && execute_if.data.rs2_data[i][31]}}, execute_if.data.rs2_data[i][31:0]}: execute_if.data.rs2_data[i]; + `else + assign div_in1[i] = execute_if.data.rs1_data[i]; + assign div_in2[i] = execute_if.data.rs2_data[i]; + `endif end `ifdef IDIV_DPI diff --git a/hw/rtl/core/VX_operands.sv b/hw/rtl/core/VX_operands.sv index 3d2c570c..ee0c493b 100644 --- a/hw/rtl/core/VX_operands.sv +++ b/hw/rtl/core/VX_operands.sv @@ -26,6 +26,7 @@ module VX_operands import VX_gpu_pkg::*; #( ); `UNUSED_PARAM (CORE_ID) localparam DATAW = `UUID_WIDTH + ISSUE_WIS_W + `NUM_THREADS + `XLEN + 1 + `EX_BITS + `INST_OP_BITS + `INST_MOD_BITS + 1 + 1 + `XLEN + `NR_BITS; + localparam RAM_ADDRW = `LOG2UP(`NUM_REGS * ISSUE_RATIO); localparam STATE_IDLE = 2'd0; localparam STATE_FETCH1 = 2'd1; @@ -38,14 +39,19 @@ module VX_operands import VX_gpu_pkg::*; #( reg [`NR_BITS-1:0] gpr_rd_rid, gpr_rd_rid_n; reg [ISSUE_WIS_W-1:0] gpr_rd_wis, gpr_rd_wis_n; - reg [ISSUE_RATIO-1:0][`NUM_THREADS-1:0][`XLEN-1:0] cache_data, cache_data_n; - reg [ISSUE_RATIO-1:0][`NR_BITS-1:0] cache_reg, cache_reg_n; - reg [ISSUE_RATIO-1:0][`NUM_THREADS-1:0] cache_tmask, cache_tmask_n; + reg [`NUM_THREADS-1:0][`XLEN-1:0] cache_data [ISSUE_RATIO-1:0]; + reg [`NUM_THREADS-1:0][`XLEN-1:0] cache_data_n [ISSUE_RATIO-1:0]; + reg [`NR_BITS-1:0] cache_reg [ISSUE_RATIO-1:0]; + reg [`NR_BITS-1:0] cache_reg_n [ISSUE_RATIO-1:0]; + reg [`NUM_THREADS-1:0] cache_tmask [ISSUE_RATIO-1:0]; + reg [`NUM_THREADS-1:0] cache_tmask_n [ISSUE_RATIO-1:0]; reg [ISSUE_RATIO-1:0] cache_eop, cache_eop_n; + reg valid_out_r; + reg [DATAW-1:0] data_out_r; reg [`NUM_THREADS-1:0][`XLEN-1:0] rs1_data, rs1_data_n; reg [`NUM_THREADS-1:0][`XLEN-1:0] rs2_data, rs2_data_n; - reg [`NUM_THREADS-1:0][`XLEN-1:0] rs3_data, rs3_data_n; + reg [`NUM_THREADS-1:0][`XLEN-1:0] rs3_data, rs3_data_n; reg [STATE_BITS-1:0] state, state_n; reg [`NR_BITS-1:0] rs2, rs2_n; @@ -54,11 +60,11 @@ module VX_operands import VX_gpu_pkg::*; #( reg rs3_ready, rs3_ready_n; reg data_ready, data_ready_n; + wire ready_out = operands_if[i].ready; + wire is_rs1_zero = (scoreboard_if[i].data.rs1 == 0); wire is_rs2_zero = (scoreboard_if[i].data.rs2 == 0); - wire is_rs3_zero = (scoreboard_if[i].data.rs3 == 0); - - VX_operands_if staging_if(); + wire is_rs3_zero = (scoreboard_if[i].data.rs3 == 0); always @(*) begin state_n = state; @@ -79,7 +85,7 @@ module VX_operands import VX_gpu_pkg::*; #( case (state) STATE_IDLE: begin - if (staging_if.valid && staging_if.ready) begin + if (valid_out_r && ready_out) begin data_ready_n = 0; end if (scoreboard_if[i].valid && data_ready_n == 0) begin @@ -160,44 +166,93 @@ module VX_operands import VX_gpu_pkg::*; #( end cache_reg_n[writeback_if[i].data.wis] = writeback_if[i].data.rd; cache_eop_n[writeback_if[i].data.wis] = writeback_if[i].data.eop; - if (writeback_if[i].data.sop) begin - cache_tmask_n[writeback_if[i].data.wis] = writeback_if[i].data.tmask; - end else begin - cache_tmask_n[writeback_if[i].data.wis] |= writeback_if[i].data.tmask; - end + cache_tmask_n[writeback_if[i].data.wis] = writeback_if[i].data.sop ? writeback_if[i].data.tmask : + (cache_tmask_n[writeback_if[i].data.wis] | writeback_if[i].data.tmask); end end end always @(posedge clk) begin - if (reset) begin + if (reset) begin state <= STATE_IDLE; - gpr_rd_rid <= '0; - gpr_rd_wis <= '0; cache_eop <= {ISSUE_RATIO{1'b1}}; - cache_reg <= '0; data_ready <= 0; + valid_out_r <= 0; end else begin state <= state_n; - rs2 <= rs2_n; - rs3 <= rs3_n; - rs2_ready <= rs2_ready_n; - rs3_ready <= rs3_ready_n; - rs1_data <= rs1_data_n; - rs2_data <= rs2_data_n; - rs3_data <= rs3_data_n; - gpr_rd_rid <= gpr_rd_rid_n; - gpr_rd_wis <= gpr_rd_wis_n; - cache_data <= cache_data_n; - cache_reg <= cache_reg_n; - cache_tmask <= cache_tmask_n; cache_eop <= cache_eop_n; - data_ready <= data_ready_n; + data_ready <= data_ready_n; + if (~valid_out_r) begin + valid_out_r <= scoreboard_if[i].valid && data_ready; + end else if (ready_out) begin + valid_out_r <= 0; + end end - end + + if (~valid_out_r) begin + data_out_r <= {scoreboard_if[i].data.uuid, + scoreboard_if[i].data.wis, + scoreboard_if[i].data.tmask, + scoreboard_if[i].data.PC, + scoreboard_if[i].data.wb, + scoreboard_if[i].data.ex_type, + scoreboard_if[i].data.op_type, + scoreboard_if[i].data.op_mod, + scoreboard_if[i].data.use_PC, + scoreboard_if[i].data.use_imm, + scoreboard_if[i].data.imm, + scoreboard_if[i].data.rd}; + end + + gpr_rd_rid <= gpr_rd_rid_n; + gpr_rd_wis <= gpr_rd_wis_n; + rs2_ready <= rs2_ready_n; + rs3_ready <= rs3_ready_n; + rs2 <= rs2_n; + rs3 <= rs3_n; + rs1_data <= rs1_data_n; + rs2_data <= rs2_data_n; + rs3_data <= rs3_data_n; + cache_data <= cache_data_n; + cache_reg <= cache_reg_n; + cache_tmask <= cache_tmask_n; + end + + assign operands_if[i].valid = valid_out_r; + assign {operands_if[i].data.uuid, + operands_if[i].data.wis, + operands_if[i].data.tmask, + operands_if[i].data.PC, + operands_if[i].data.wb, + operands_if[i].data.ex_type, + operands_if[i].data.op_type, + operands_if[i].data.op_mod, + operands_if[i].data.use_PC, + operands_if[i].data.use_imm, + operands_if[i].data.imm, + operands_if[i].data.rd} = data_out_r; + assign operands_if[i].data.rs1_data = rs1_data; + assign operands_if[i].data.rs2_data = rs2_data; + assign operands_if[i].data.rs3_data = rs3_data; + + assign scoreboard_if[i].ready = ~valid_out_r && data_ready; // GPR banks + reg [RAM_ADDRW-1:0] gpr_rd_addr; + wire [RAM_ADDRW-1:0] gpr_wr_addr; + if (ISSUE_WIS != 0) begin + assign gpr_wr_addr = {writeback_if[i].data.wis, writeback_if[i].data.rd}; + always @(posedge clk) begin + gpr_rd_addr <= {gpr_rd_wis_n, gpr_rd_rid_n}; + end + end else begin + assign gpr_wr_addr = writeback_if[i].data.rd; + always @(posedge clk) begin + gpr_rd_addr <= gpr_rd_rid_n; + end + end + `ifdef GPR_RESET reg wr_enabled = 0; always @(posedge clk) begin @@ -205,10 +260,8 @@ module VX_operands import VX_gpu_pkg::*; #( wr_enabled <= 1; end end - `else - wire wr_enabled = 1; `endif - + for (genvar j = 0; j < `NUM_THREADS; ++j) begin VX_dp_ram #( .DATAW (`XLEN), @@ -222,81 +275,17 @@ module VX_operands import VX_gpu_pkg::*; #( .clk (clk), .read (1'b1), `UNUSED_PIN (wren), - .write (wr_enabled && writeback_if[i].valid && writeback_if[i].data.tmask[j]), - .waddr (wis_to_addr(writeback_if[i].data.rd, writeback_if[i].data.wis)), + `ifdef GPR_RESET + .write (wr_enabled && writeback_if[i].valid && writeback_if[i].data.tmask[j]), + `else + .write (writeback_if[i].valid && writeback_if[i].data.tmask[j]), + `endif + .waddr (gpr_wr_addr), .wdata (writeback_if[i].data.data[j]), - .raddr (wis_to_addr(gpr_rd_rid, gpr_rd_wis)), + .raddr (gpr_rd_addr), .rdata (gpr_rd_data[j]) ); end - - // staging buffer - - `RESET_RELAY (stg_buf_reset, reset); - - VX_elastic_buffer #( - .DATAW (DATAW) - ) stg_buf ( - .clk (clk), - .reset (stg_buf_reset), - .valid_in (scoreboard_if[i].valid), - .ready_in (scoreboard_if[i].ready), - .data_in ({ - scoreboard_if[i].data.uuid, - scoreboard_if[i].data.wis, - scoreboard_if[i].data.tmask, - scoreboard_if[i].data.PC, - scoreboard_if[i].data.wb, - scoreboard_if[i].data.ex_type, - scoreboard_if[i].data.op_type, - scoreboard_if[i].data.op_mod, - scoreboard_if[i].data.use_PC, - scoreboard_if[i].data.use_imm, - scoreboard_if[i].data.imm, - scoreboard_if[i].data.rd}), - .data_out ({ - staging_if.data.uuid, - staging_if.data.wis, - staging_if.data.tmask, - staging_if.data.PC, - staging_if.data.wb, - staging_if.data.ex_type, - staging_if.data.op_type, - staging_if.data.op_mod, - staging_if.data.use_PC, - staging_if.data.use_imm, - staging_if.data.imm, - staging_if.data.rd}), - .valid_out (staging_if.valid), - .ready_out (staging_if.ready) - ); - - assign staging_if.data.rs1_data = rs1_data; - assign staging_if.data.rs2_data = rs2_data; - assign staging_if.data.rs3_data = rs3_data; - - // output buffer - - wire valid_stg, ready_stg; - assign valid_stg = staging_if.valid && data_ready; - assign staging_if.ready = ready_stg && data_ready; - - `RESET_RELAY (out_buf_reset, reset); - - VX_elastic_buffer #( - .DATAW (DATAW + (3 * `NUM_THREADS * `XLEN)), - .SIZE (2), - .OUT_REG (2) - ) out_buf ( - .clk (clk), - .reset (out_buf_reset), - .valid_in (valid_stg), - .ready_in (ready_stg), - .data_in (staging_if.data), - .data_out (operands_if[i].data), - .valid_out (operands_if[i].valid), - .ready_out (operands_if[i].ready) - ); - end + end endmodule diff --git a/hw/rtl/core/VX_schedule.sv b/hw/rtl/core/VX_schedule.sv index ea96178e..36bdf9e0 100644 --- a/hw/rtl/core/VX_schedule.sv +++ b/hw/rtl/core/VX_schedule.sv @@ -19,6 +19,10 @@ module VX_schedule import VX_gpu_pkg::*; #( input wire clk, input wire reset, +`ifdef PERF_ENABLE + VX_pipeline_perf_if.schedule perf_schedule_if, +`endif + // configuration input base_dcrs_t base_dcrs, @@ -304,13 +308,20 @@ module VX_schedule import VX_gpu_pkg::*; #( localparam GNW_WIDTH = `LOG2UP(`NUM_CLUSTERS * `NUM_CORES * `NUM_WARPS); reg [`UUID_WIDTH-1:0] instr_uuid; wire [GNW_WIDTH-1:0] g_wid = (GNW_WIDTH'(CORE_ID) << `NW_BITS) + GNW_WIDTH'(schedule_wid); +`ifdef SV_DPI always @(posedge clk) begin if (reset) begin instr_uuid <= `UUID_WIDTH'(dpi_uuid_gen(1, 0, 0)); end else if (schedule_fire) begin instr_uuid <= `UUID_WIDTH'(dpi_uuid_gen(0, 32'(g_wid), 64'(schedule_pc))); - end + end end +`else + wire [GNW_WIDTH+16-1:0] w_uuid = {g_wid, 16'(schedule_pc)}; + always @(*) begin + instr_uuid = `UUID_WIDTH'(w_uuid); + end +`endif `else wire [`UUID_WIDTH-1:0] instr_uuid = '0; `endif @@ -349,7 +360,7 @@ module VX_schedule import VX_gpu_pkg::*; #( .empty (no_pending_instr) ); - `BUFFER_BUSY (busy, (active_warps != 0 || ~no_pending_instr), 1); + `BUFFER_EX(busy, (active_warps != 0 || ~no_pending_instr), 1'b1, 1); // export CSRs assign sched_csr_if.cycles = cycles; @@ -376,4 +387,25 @@ module VX_schedule import VX_gpu_pkg::*; #( end `RUNTIME_ASSERT(timeout_ctr < `STALL_TIMEOUT, ("%t: *** core%0d-scheduler-timeout: stalled_warps=%b", $time, CORE_ID, stalled_warps)); +`ifdef PERF_ENABLE + reg [`PERF_CTR_BITS-1:0] perf_sched_idles; + reg [`PERF_CTR_BITS-1:0] perf_sched_stalls; + + wire schedule_idle = ~schedule_valid; + wire schedule_stall = schedule_if.valid && ~schedule_if.ready; + + always @(posedge clk) begin + if (reset) begin + perf_sched_idles <= '0; + perf_sched_stalls <= '0; + end else begin + perf_sched_idles <= perf_sched_idles + `PERF_CTR_BITS'(schedule_idle); + perf_sched_stalls <= perf_sched_stalls + `PERF_CTR_BITS'(schedule_stall); + end + end + + assign perf_schedule_if.sched_idles = perf_sched_idles; + assign perf_schedule_if.sched_stalls = perf_sched_stalls; +`endif + endmodule diff --git a/hw/rtl/core/VX_scoreboard.sv b/hw/rtl/core/VX_scoreboard.sv index ee5ae2ec..a4792c8d 100644 --- a/hw/rtl/core/VX_scoreboard.sv +++ b/hw/rtl/core/VX_scoreboard.sv @@ -19,6 +19,12 @@ module VX_scoreboard import VX_gpu_pkg::*; #( input wire clk, input wire reset, +`ifdef PERF_ENABLE + output reg [`PERF_CTR_BITS-1:0] perf_scb_stalls, + output reg [`PERF_CTR_BITS-1:0] perf_units_uses [`NUM_EX_UNITS], + output reg [`PERF_CTR_BITS-1:0] perf_sfu_uses [`NUM_SFU_UNITS], +`endif + VX_writeback_if.slave writeback_if [`ISSUE_WIDTH], VX_ibuffer_if.slave ibuffer_if [`ISSUE_WIDTH], VX_ibuffer_if.master scoreboard_if [`ISSUE_WIDTH] @@ -26,114 +32,201 @@ module VX_scoreboard import VX_gpu_pkg::*; #( `UNUSED_PARAM (CORE_ID) localparam DATAW = `UUID_WIDTH + ISSUE_WIS_W + `NUM_THREADS + `XLEN + `EX_BITS + `INST_OP_BITS + `INST_MOD_BITS + 1 + 1 + `XLEN + (`NR_BITS * 4) + 1; +`ifdef PERF_ENABLE + reg [`ISSUE_WIDTH-1:0][`NUM_EX_UNITS-1:0] perf_issue_units_per_cycle; + wire [`NUM_EX_UNITS-1:0] perf_units_per_cycle, perf_units_per_cycle_r; + + reg [`ISSUE_WIDTH-1:0][`NUM_SFU_UNITS-1:0] perf_issue_sfu_per_cycle; + wire [`NUM_SFU_UNITS-1:0] perf_sfu_per_cycle, perf_sfu_per_cycle_r; + + wire [`ISSUE_WIDTH-1:0] perf_issue_stalls_per_cycle; + wire [`CLOG2(`ISSUE_WIDTH+1)-1:0] perf_stalls_per_cycle, perf_stalls_per_cycle_r; + + `POP_COUNT(perf_stalls_per_cycle, perf_issue_stalls_per_cycle); + + VX_reduce #( + .DATAW_IN (`NUM_EX_UNITS), + .N (`ISSUE_WIDTH), + .OP ("|") + ) perf_units_reduce ( + .data_in (perf_issue_units_per_cycle), + .data_out (perf_units_per_cycle) + ); + + VX_reduce #( + .DATAW_IN (`NUM_SFU_UNITS), + .N (`ISSUE_WIDTH), + .OP ("|") + ) perf_sfu_reduce ( + .data_in (perf_issue_sfu_per_cycle), + .data_out (perf_sfu_per_cycle) + ); + + `BUFFER(perf_stalls_per_cycle_r, perf_stalls_per_cycle); + `BUFFER(perf_units_per_cycle_r, perf_units_per_cycle); + `BUFFER(perf_sfu_per_cycle_r, perf_sfu_per_cycle); + + always @(posedge clk) begin + if (reset) begin + perf_scb_stalls <= '0; + end else begin + perf_scb_stalls <= perf_scb_stalls + `PERF_CTR_BITS'(perf_stalls_per_cycle_r); + end + end + + for (genvar i = 0; i < `NUM_EX_UNITS; ++i) begin + always @(posedge clk) begin + if (reset) begin + perf_units_uses[i] <= '0; + end else begin + perf_units_uses[i] <= perf_units_uses[i] + `PERF_CTR_BITS'(perf_units_per_cycle_r[i]); + end + end + end + + for (genvar i = 0; i < `NUM_SFU_UNITS; ++i) begin + always @(posedge clk) begin + if (reset) begin + perf_sfu_uses[i] <= '0; + end else begin + perf_sfu_uses[i] <= perf_sfu_uses[i] + `PERF_CTR_BITS'(perf_sfu_per_cycle_r[i]); + end + end + end +`endif + for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin - reg [`UP(ISSUE_RATIO)-1:0][`NUM_REGS-1:0] inuse_regs, inuse_regs_n; - reg [3:0] ready_masks, ready_masks_n; - VX_ibuffer_if staging_if(); - + reg [`UP(ISSUE_RATIO)-1:0][`NUM_REGS-1:0] inuse_regs; + wire writeback_fire = writeback_if[i].valid && writeback_if[i].data.eop; + wire inuse_rd = inuse_regs[ibuffer_if[i].data.wis][ibuffer_if[i].data.rd]; + wire inuse_rs1 = inuse_regs[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs1]; + wire inuse_rs2 = inuse_regs[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs2]; + wire inuse_rs3 = inuse_regs[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs3]; + + `ifdef PERF_ENABLE + reg [`UP(ISSUE_RATIO)-1:0][`NUM_REGS-1:0][`EX_WIDTH-1:0] inuse_units; + reg [`UP(ISSUE_RATIO)-1:0][`NUM_REGS-1:0][`SFU_WIDTH-1:0] inuse_sfu; + + reg [`SFU_WIDTH-1:0] sfu_type; always @(*) begin - inuse_regs_n = inuse_regs; - ready_masks_n = ready_masks; - if (writeback_fire) begin - inuse_regs_n[writeback_if[i].data.wis][writeback_if[i].data.rd] = 0; - ready_masks_n |= {4{(ISSUE_RATIO == 0) || writeback_if[i].data.wis == staging_if.data.wis}} - & {(writeback_if[i].data.rd == staging_if.data.rd), - (writeback_if[i].data.rd == staging_if.data.rs1), - (writeback_if[i].data.rd == staging_if.data.rs2), - (writeback_if[i].data.rd == staging_if.data.rs3)}; - end - if (staging_if.valid && staging_if.ready && staging_if.data.wb) begin - inuse_regs_n[staging_if.data.wis][staging_if.data.rd] = 1; - ready_masks_n = '0; + case (scoreboard_if[i].data.op_type) + `INST_SFU_CSRRW, + `INST_SFU_CSRRS, + `INST_SFU_CSRRC: sfu_type = `SFU_CSRS; + default: sfu_type = `SFU_WCTL; + endcase + end + + always @(*) begin + perf_issue_units_per_cycle[i] = '0; + perf_issue_sfu_per_cycle[i] = '0; + if (ibuffer_if[i].valid) begin + if (inuse_rd) begin + perf_issue_units_per_cycle[i][inuse_units[ibuffer_if[i].data.wis][ibuffer_if[i].data.rd]] = 1; + if (inuse_units[ibuffer_if[i].data.wis][ibuffer_if[i].data.rd] == `EX_SFU) begin + perf_issue_sfu_per_cycle[i][inuse_sfu[ibuffer_if[i].data.wis][ibuffer_if[i].data.rd]] = 1; + end + end + if (inuse_rs1) begin + perf_issue_units_per_cycle[i][inuse_units[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs1]] = 1; + if (inuse_units[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs1] == `EX_SFU) begin + perf_issue_sfu_per_cycle[i][inuse_sfu[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs1]] = 1; + end + end + if (inuse_rs2) begin + perf_issue_units_per_cycle[i][inuse_units[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs2]] = 1; + if (inuse_units[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs2] == `EX_SFU) begin + perf_issue_sfu_per_cycle[i][inuse_sfu[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs2]] = 1; + end + end + if (inuse_rs3) begin + perf_issue_units_per_cycle[i][inuse_units[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs3]] = 1; + if (inuse_units[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs3] == `EX_SFU) begin + perf_issue_sfu_per_cycle[i][inuse_sfu[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs3]] = 1; + end + end end - if (ibuffer_if[i].valid && ibuffer_if[i].ready) begin - ready_masks_n = ~{inuse_regs_n[ibuffer_if[i].data.wis][ibuffer_if[i].data.rd], - inuse_regs_n[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs1], - inuse_regs_n[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs2], - inuse_regs_n[ibuffer_if[i].data.wis][ibuffer_if[i].data.rs3]}; - end - end + end + assign perf_issue_stalls_per_cycle[i] = ibuffer_if[i].valid && ~ibuffer_if[i].ready; + `endif + + reg [DATAW-1:0] data_out_r; + reg valid_out_r; + wire ready_out; + + wire [3:0] ready_masks = ~{inuse_rd, inuse_rs1, inuse_rs2, inuse_rs3}; + wire deps_ready = (& ready_masks); + + wire valid_in = ibuffer_if[i].valid && deps_ready; + wire ready_in = ~valid_out_r && deps_ready; + wire [DATAW-1:0] data_in = ibuffer_if[i].data; + + assign ready_out = scoreboard_if[i].ready; always @(posedge clk) begin if (reset) begin - inuse_regs <= '0; - ready_masks <= '0; - end else begin - inuse_regs <= inuse_regs_n; - ready_masks <= ready_masks_n; + valid_out_r <= 0; + inuse_regs <= '0; + end else begin + if (writeback_fire) begin + inuse_regs[writeback_if[i].data.wis][writeback_if[i].data.rd] <= 0; + end + if (~valid_out_r) begin + valid_out_r <= valid_in; + end else if (ready_out) begin + if (scoreboard_if[i].data.wb) begin + inuse_regs[scoreboard_if[i].data.wis][scoreboard_if[i].data.rd] <= 1; + `ifdef PERF_ENABLE + inuse_units[scoreboard_if[i].data.wis][scoreboard_if[i].data.rd] <= scoreboard_if[i].data.ex_type; + if (scoreboard_if[i].data.ex_type == `EX_SFU) begin + inuse_sfu[scoreboard_if[i].data.wis][scoreboard_if[i].data.rd] <= sfu_type; + end + `endif + end + valid_out_r <= 0; + end + end + if (~valid_out_r) begin + data_out_r <= data_in; end end - // staging buffer + assign ibuffer_if[i].ready = ready_in; + assign scoreboard_if[i].valid = valid_out_r; + assign scoreboard_if[i].data = data_out_r; - `RESET_RELAY (stg_buf_reset, reset); - - VX_elastic_buffer #( - .DATAW (DATAW) - ) stg_buf ( - .clk (clk), - .reset (stg_buf_reset), - .valid_in (ibuffer_if[i].valid), - .ready_in (ibuffer_if[i].ready), - .data_in (ibuffer_if[i].data), - .data_out (staging_if.data), - .valid_out (staging_if.valid), - .ready_out (staging_if.ready) - ); + `ifdef SIMULATION + reg [31:0] timeout_ctr; - // output buffer - - wire valid_stg, ready_stg; - wire regs_ready = (& ready_masks); - assign valid_stg = staging_if.valid && regs_ready; - assign staging_if.ready = ready_stg && regs_ready; - - `RESET_RELAY (out_buf_reset, reset); - - VX_elastic_buffer #( - .DATAW (DATAW), - .SIZE (2), - .OUT_REG (2) - ) out_buf ( - .clk (clk), - .reset (out_buf_reset), - .valid_in (valid_stg), - .ready_in (ready_stg), - .data_in (staging_if.data), - .data_out (scoreboard_if[i].data), - .valid_out (scoreboard_if[i].valid), - .ready_out (scoreboard_if[i].ready) - ); - - reg [31:0] timeout_ctr; - always @(posedge clk) begin if (reset) begin timeout_ctr <= '0; end else begin - if (staging_if.valid && ~regs_ready) begin + if (ibuffer_if[i].valid && ~ibuffer_if[i].ready) begin `ifdef DBG_TRACE_CORE_PIPELINE `TRACE(3, ("%d: *** core%0d-scoreboard-stall: wid=%0d, PC=0x%0h, tmask=%b, cycles=%0d, inuse=%b (#%0d)\n", - $time, CORE_ID, wis_to_wid(staging_if.data.wis, i), staging_if.data.PC, staging_if.data.tmask, timeout_ctr, - ~ready_masks, staging_if.data.uuid)); + $time, CORE_ID, wis_to_wid(ibuffer_if[i].data.wis, i), ibuffer_if[i].data.PC, ibuffer_if[i].data.tmask, timeout_ctr, + ~ready_masks, ibuffer_if[i].data.uuid)); `endif timeout_ctr <= timeout_ctr + 1; - end else if (staging_if.valid && staging_if.ready) begin + end else if (ibuffer_if[i].valid && ibuffer_if[i].ready) begin timeout_ctr <= '0; end end end - + `RUNTIME_ASSERT((timeout_ctr < `STALL_TIMEOUT), ("%t: *** core%0d-scoreboard-timeout: wid=%0d, PC=0x%0h, tmask=%b, cycles=%0d, inuse=%b (#%0d)", - $time, CORE_ID, wis_to_wid(staging_if.data.wis, i), staging_if.data.PC, staging_if.data.tmask, timeout_ctr, - ~ready_masks, staging_if.data.uuid)); + $time, CORE_ID, wis_to_wid(ibuffer_if[i].data.wis, i), ibuffer_if[i].data.PC, ibuffer_if[i].data.tmask, timeout_ctr, + ~ready_masks, ibuffer_if[i].data.uuid)); `RUNTIME_ASSERT(~writeback_fire || inuse_regs[writeback_if[i].data.wis][writeback_if[i].data.rd] != 0, ("%t: *** core%0d: invalid writeback register: wid=%0d, PC=0x%0h, tmask=%b, rd=%0d (#%0d)", $time, CORE_ID, wis_to_wid(writeback_if[i].data.wis, i), writeback_if[i].data.PC, writeback_if[i].data.tmask, writeback_if[i].data.rd, writeback_if[i].data.uuid)); - end + `endif + + end endmodule diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index e94f86fd..6fb2cb9f 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -48,7 +48,7 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( localparam RSP_ARB_DATAW = `UUID_WIDTH + `NW_WIDTH + NUM_LANES + (NUM_LANES * `XLEN) + `NR_BITS + 1 + `XLEN + PID_WIDTH + 1 + 1; localparam RSP_ARB_SIZE = 1 + 1; localparam RSP_ARB_IDX_WCTL = 0; - localparam RSP_ARB_IDX_CSR = 1; + localparam RSP_ARB_IDX_CSRS = 1; VX_execute_if #( .NUM_LANES (NUM_LANES) @@ -71,9 +71,6 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( wire [RSP_ARB_SIZE-1:0] rsp_arb_ready_in; wire [RSP_ARB_SIZE-1:0][RSP_ARB_DATAW-1:0] rsp_arb_data_in; -`ifdef PERF_ENABLE - VX_sfu_perf_if sfu_perf_if(); -`endif // Warp control block VX_execute_if #( @@ -129,7 +126,6 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( `ifdef PERF_ENABLE .mem_perf_if (mem_perf_if), .pipeline_perf_if(pipeline_perf_if), - .sfu_perf_if (sfu_perf_if), `endif `ifdef EXT_F_ENABLE @@ -141,21 +137,21 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( .commit_if (csr_commit_if) ); - assign rsp_arb_valid_in[RSP_ARB_IDX_CSR] = csr_commit_if.valid; - assign rsp_arb_data_in[RSP_ARB_IDX_CSR] = csr_commit_if.data; - assign csr_commit_if.ready = rsp_arb_ready_in[RSP_ARB_IDX_CSR]; + assign rsp_arb_valid_in[RSP_ARB_IDX_CSRS] = csr_commit_if.valid; + assign rsp_arb_data_in[RSP_ARB_IDX_CSRS] = csr_commit_if.data; + assign csr_commit_if.ready = rsp_arb_ready_in[RSP_ARB_IDX_CSRS]; // can accept new request? reg sfu_req_ready; always @(*) begin case (execute_if[0].data.op_type) - `INST_SFU_CSRRW, - `INST_SFU_CSRRS, - `INST_SFU_CSRRC: sfu_req_ready = csr_execute_if.ready; + `INST_SFU_CSRRW, + `INST_SFU_CSRRS, + `INST_SFU_CSRRC: sfu_req_ready = csr_execute_if.ready; default: sfu_req_ready = wctl_execute_if.ready; endcase - end + end assign execute_if[0].ready = sfu_req_ready; // response arbitration @@ -170,7 +166,7 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( .NUM_INPUTS (RSP_ARB_SIZE), .DATAW (RSP_ARB_DATAW), .ARBITER ("R"), - .OUT_REG (1) + .OUT_REG (3) ) rsp_arb ( .clk (clk), .reset (commit_reset), @@ -186,7 +182,7 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( VX_gather_unit #( .BLOCK_SIZE (BLOCK_SIZE), .NUM_LANES (NUM_LANES), - .OUT_REG (3) + .OUT_REG (1) ) gather_unit ( .clk (clk), .reset (commit_reset), @@ -194,16 +190,4 @@ module VX_sfu_unit import VX_gpu_pkg::*; #( .commit_out_if (commit_if) ); -`ifdef PERF_ENABLE - reg [`PERF_CTR_BITS-1:0] perf_wctl_stalls; - always @(posedge clk) begin - if (reset) begin - perf_wctl_stalls <= '0; - end else begin - perf_wctl_stalls <= perf_wctl_stalls + `PERF_CTR_BITS'(wctl_execute_if.valid && ~wctl_execute_if.ready); - end - end - assign sfu_perf_if.wctl_stalls = perf_wctl_stalls; -`endif - endmodule diff --git a/hw/rtl/core/VX_trace.vh b/hw/rtl/core/VX_trace.vh index 1dea3347..ff2b3bb9 100644 --- a/hw/rtl/core/VX_trace.vh +++ b/hw/rtl/core/VX_trace.vh @@ -14,9 +14,7 @@ `ifndef VX_TRACE_VH `define VX_TRACE_VH -`ifndef SYNTHESIS - -`include "VX_define.vh" +`ifdef SIMULATION task trace_ex_type(input int level, input [`EX_BITS-1:0] ex_type); case (ex_type) diff --git a/hw/rtl/core/VX_wctl_unit.sv b/hw/rtl/core/VX_wctl_unit.sv index d628e16c..88b2f71e 100644 --- a/hw/rtl/core/VX_wctl_unit.sv +++ b/hw/rtl/core/VX_wctl_unit.sv @@ -29,7 +29,6 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( ); `UNUSED_PARAM (CORE_ID) localparam LANE_BITS = `CLOG2(NUM_LANES); - localparam LANE_WIDTH = `UP(LANE_BITS); localparam PID_BITS = `CLOG2(`NUM_THREADS / NUM_LANES); localparam PID_WIDTH = `UP(PID_BITS); localparam WCTL_WIDTH = $bits(tmc_t) + $bits(wspawn_t) + $bits(split_t) + $bits(join_t) + $bits(barrier_t); @@ -50,7 +49,7 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( wire is_join = (execute_if.data.op_type == `INST_SFU_JOIN); wire is_bar = (execute_if.data.op_type == `INST_SFU_BAR); - wire [LANE_WIDTH-1:0] tid; + wire [`UP(LANE_BITS)-1:0] tid; if (LANE_BITS != 0) begin assign tid = execute_if.data.tid[0 +: LANE_BITS]; end else begin diff --git a/hw/rtl/fpu/VX_fpu_cvt.sv b/hw/rtl/fpu/VX_fpu_cvt.sv index 34b2ed28..7ba6330e 100644 --- a/hw/rtl/fpu/VX_fpu_cvt.sv +++ b/hw/rtl/fpu/VX_fpu_cvt.sv @@ -52,30 +52,24 @@ module VX_fpu_cvt import VX_fpu_pkg::*; #( localparam MAN_BITS = 23; localparam EXP_BITS = 8; - localparam EXP_BIAS = 2**(EXP_BITS-1)-1; - - localparam logic [EXP_BITS-1:0] QNAN_EXPONENT = 2**EXP_BITS-1; - localparam logic [MAN_BITS-1:0] QNAN_MANTISSA = 2**(MAN_BITS-1); + localparam EXP_BIAS = 2**(EXP_BITS-1)-1; // Use 32-bit integer - localparam MAX_INT_WIDTH = 32; + localparam INT_WIDTH = 32; // The internal mantissa includes normal bit or an entire integer - localparam INT_MAN_WIDTH = `MAX(MAN_BITS + 1, MAX_INT_WIDTH); + localparam INT_MAN_WIDTH = `MAX(MAN_BITS + 1, INT_WIDTH); // The lower 2p+3 bits of the internal FMA result will be needed for leading-zero detection localparam LZC_RESULT_WIDTH = `CLOG2(INT_MAN_WIDTH); // The internal exponent must be able to represent the smallest denormal input value as signed // or the number of bits in an integer - localparam INT_EXP_WIDTH = `MAX(`CLOG2(MAX_INT_WIDTH), `MAX(EXP_BITS, `CLOG2(EXP_BIAS + MAN_BITS))) + 1; - - // shift amount for denormalization - localparam SHAMT_BITS = `CLOG2(INT_MAN_WIDTH+1); + localparam INT_EXP_WIDTH = `MAX(`CLOG2(INT_WIDTH), `MAX(EXP_BITS, `CLOG2(EXP_BIAS + MAN_BITS))) + 1; localparam FMT_SHIFT_COMPENSATION = INT_MAN_WIDTH - 1 - MAN_BITS; localparam NUM_FP_STICKY = 2 * INT_MAN_WIDTH - MAN_BITS - 1; // removed mantissa, 1. and R - localparam NUM_INT_STICKY = 2 * INT_MAN_WIDTH - MAX_INT_WIDTH; // removed int and R + localparam NUM_INT_STICKY = 2 * INT_MAN_WIDTH - INT_WIDTH; // removed int and R // Input processing @@ -86,8 +80,8 @@ module VX_fpu_cvt import VX_fpu_pkg::*; #( .EXP_BITS (EXP_BITS), .MAN_BITS (MAN_BITS) ) fp_class ( - .exp_i (dataa[i][30:23]), - .man_i (dataa[i][22:0]), + .exp_i (dataa[i][INT_WIDTH-2:MAN_BITS]), + .man_i (dataa[i][MAN_BITS-1:0]), .clss_o (fclass[i]) ); end @@ -97,27 +91,25 @@ module VX_fpu_cvt import VX_fpu_pkg::*; #( wire [NUM_LANES-1:0] input_sign; for (genvar i = 0; i < NUM_LANES; ++i) begin - wire [INT_MAN_WIDTH-1:0] int_mantissa; - wire [INT_MAN_WIDTH-1:0] fmt_mantissa; - wire fmt_sign = dataa[i][31]; - wire int_sign = dataa[i][31] && is_signed; - assign int_mantissa = int_sign ? (-dataa[i]) : dataa[i]; - assign fmt_mantissa = INT_MAN_WIDTH'({fclass[i].is_normal, dataa[i][MAN_BITS-1:0]}); + wire i2f_sign = dataa[i][INT_WIDTH-1]; + wire f2i_sign = dataa[i][INT_WIDTH-1] && is_signed; + wire [INT_MAN_WIDTH-1:0] f2i_mantissa = f2i_sign ? (-dataa[i]) : dataa[i]; + wire [INT_MAN_WIDTH-1:0] i2f_mantissa = INT_MAN_WIDTH'({fclass[i].is_normal, dataa[i][MAN_BITS-1:0]}); assign input_exp[i] = {1'b0, dataa[i][MAN_BITS +: EXP_BITS]} + INT_EXP_WIDTH'({1'b0, fclass[i].is_subnormal}); - assign input_mant[i] = is_itof ? int_mantissa : fmt_mantissa; - assign input_sign[i] = is_itof ? int_sign : fmt_sign; + assign input_mant[i] = is_itof ? f2i_mantissa : i2f_mantissa; + assign input_sign[i] = is_itof ? f2i_sign : i2f_sign; end // Pipeline stage0 - wire valid_in_s0; - wire [NUM_LANES-1:0] lane_mask_s0; - wire [TAGW-1:0] tag_in_s0; - wire is_itof_s0; - wire unsigned_s0; - wire [2:0] rnd_mode_s0; + wire valid_in_s0; + wire [NUM_LANES-1:0] lane_mask_s0; + wire [TAGW-1:0] tag_in_s0; + wire is_itof_s0; + wire is_signed_s0; + wire [2:0] rnd_mode_s0; fclass_t [NUM_LANES-1:0] fclass_s0; - wire [NUM_LANES-1:0] input_sign_s0; + wire [NUM_LANES-1:0] input_sign_s0; wire [NUM_LANES-1:0][INT_EXP_WIDTH-1:0] fmt_exponent_s0; wire [NUM_LANES-1:0][INT_MAN_WIDTH-1:0] encoded_mant_s0; @@ -130,8 +122,8 @@ module VX_fpu_cvt import VX_fpu_pkg::*; #( .clk (clk), .reset (reset), .enable (~stall), - .data_in ({valid_in, lane_mask, tag_in, is_itof, !is_signed, frm, fclass, input_sign, input_exp, input_mant}), - .data_out ({valid_in_s0, lane_mask_s0, tag_in_s0, is_itof_s0, unsigned_s0, rnd_mode_s0, fclass_s0, input_sign_s0, fmt_exponent_s0, encoded_mant_s0}) + .data_in ({valid_in, lane_mask, tag_in, is_itof, is_signed, frm, fclass, input_sign, input_exp, input_mant}), + .data_out ({valid_in_s0, lane_mask_s0, tag_in_s0, is_itof_s0, is_signed_s0, rnd_mode_s0, fclass_s0, input_sign_s0, fmt_exponent_s0, encoded_mant_s0}) ); // Normalization @@ -159,22 +151,22 @@ module VX_fpu_cvt import VX_fpu_pkg::*; #( assign input_mant_n_s0[i] = encoded_mant_s0[i] << renorm_shamt_s0[i]; // Unbias exponent and compensate for shift - wire [INT_EXP_WIDTH-1:0] fp_input_exp_s0 = fmt_exponent_s0[i] + INT_EXP_WIDTH'(FMT_SHIFT_COMPENSATION - EXP_BIAS) - INT_EXP_WIDTH'({1'b0, renorm_shamt_s0[i]}); - wire [INT_EXP_WIDTH-1:0] int_input_exp_s0 = INT_EXP_WIDTH'(INT_MAN_WIDTH-1) - INT_EXP_WIDTH'({1'b0, renorm_shamt_s0[i]}); - assign input_exp_n_s0[i] = is_itof_s0 ? int_input_exp_s0 : fp_input_exp_s0; + wire [INT_EXP_WIDTH-1:0] i2f_input_exp_s0 = fmt_exponent_s0[i] + INT_EXP_WIDTH'(FMT_SHIFT_COMPENSATION - EXP_BIAS) - INT_EXP_WIDTH'({1'b0, renorm_shamt_s0[i]}); + wire [INT_EXP_WIDTH-1:0] f2i_input_exp_s0 = INT_EXP_WIDTH'(INT_MAN_WIDTH-1) - INT_EXP_WIDTH'({1'b0, renorm_shamt_s0[i]}); + assign input_exp_n_s0[i] = is_itof_s0 ? f2i_input_exp_s0 : i2f_input_exp_s0; end // Pipeline stage1 - wire valid_in_s1; - wire [NUM_LANES-1:0] lane_mask_s1; - wire [TAGW-1:0] tag_in_s1; - wire is_itof_s1; - wire unsigned_s1; - wire [2:0] rnd_mode_s1; + wire valid_in_s1; + wire [NUM_LANES-1:0] lane_mask_s1; + wire [TAGW-1:0] tag_in_s1; + wire is_itof_s1; + wire is_signed_s1; + wire [2:0] rnd_mode_s1; fclass_t [NUM_LANES-1:0] fclass_s1; - wire [NUM_LANES-1:0] input_sign_s1; - wire [NUM_LANES-1:0] mant_is_zero_s1; + wire [NUM_LANES-1:0] input_sign_s1; + wire [NUM_LANES-1:0] mant_is_zero_s1; wire [NUM_LANES-1:0][INT_MAN_WIDTH-1:0] input_mant_s1; wire [NUM_LANES-1:0][INT_EXP_WIDTH-1:0] input_exp_s1; @@ -185,76 +177,49 @@ module VX_fpu_cvt import VX_fpu_pkg::*; #( .clk (clk), .reset (reset), .enable (~stall), - .data_in ({valid_in_s0, lane_mask_s0, tag_in_s0, is_itof_s0, unsigned_s0, rnd_mode_s0, fclass_s0, input_sign_s0, mant_is_zero_s0, input_mant_n_s0, input_exp_n_s0}), - .data_out ({valid_in_s1, lane_mask_s1, tag_in_s1, is_itof_s1, unsigned_s1, rnd_mode_s1, fclass_s1, input_sign_s1, mant_is_zero_s1, input_mant_s1, input_exp_s1}) + .data_in ({valid_in_s0, lane_mask_s0, tag_in_s0, is_itof_s0, is_signed_s0, rnd_mode_s0, fclass_s0, input_sign_s0, mant_is_zero_s0, input_mant_n_s0, input_exp_n_s0}), + .data_out ({valid_in_s1, lane_mask_s1, tag_in_s1, is_itof_s1, is_signed_s1, rnd_mode_s1, fclass_s1, input_sign_s1, mant_is_zero_s1, input_mant_s1, input_exp_s1}) ); // Perform adjustments to mantissa and exponent wire [NUM_LANES-1:0][2*INT_MAN_WIDTH:0] destination_mant_s1; wire [NUM_LANES-1:0][INT_EXP_WIDTH-1:0] final_exp_s1; - wire [NUM_LANES-1:0] of_before_round_s1; + wire [NUM_LANES-1:0] of_before_round_s1; for (genvar i = 0; i < NUM_LANES; ++i) begin - reg [2*INT_MAN_WIDTH:0] preshift_mant_s1; // mantissa before final shift - reg [SHAMT_BITS-1:0] denorm_shamt_s1; // shift amount for denormalization - reg [INT_EXP_WIDTH-1:0] final_exp_tmp_s1; // after eventual adjustments - reg of_before_round_tmp_s1; - + wire [INT_EXP_WIDTH-1:0] denorm_shamt = INT_EXP_WIDTH'(INT_WIDTH-1) - input_exp_s1[i]; + wire overflow = ($signed(denorm_shamt) <= -$signed(INT_EXP_WIDTH'(!is_signed_s1))); + wire underflow = ($signed(input_exp_s1[i]) < INT_EXP_WIDTH'($signed(-1))); + reg [INT_EXP_WIDTH-1:0] denorm_shamt_q; always @(*) begin - final_exp_tmp_s1 = input_exp_s1[i] + INT_EXP_WIDTH'(EXP_BIAS); // take exponent as is, only look at lower bits - preshift_mant_s1 = {input_mant_s1[i], 33'b0}; - denorm_shamt_s1 = '0; - of_before_round_tmp_s1 = 1'b0; - - if (is_itof_s1) begin - if ($signed(input_exp_s1[i]) >= INT_EXP_WIDTH'($signed(2**EXP_BITS-1-EXP_BIAS))) begin - // Overflow or infinities (for proper rounding) - final_exp_tmp_s1 = (2**EXP_BITS-2); // largest normal value - preshift_mant_s1 = ~0; // largest normal value and RS bits set - of_before_round_tmp_s1 = 1'b1; - end else if ($signed(input_exp_s1[i]) < INT_EXP_WIDTH'($signed(-MAN_BITS-EXP_BIAS))) begin - // Limit the shift to retain sticky bits - final_exp_tmp_s1 = '0; // denormal result - denorm_shamt_s1 = (2 + MAN_BITS); // to sticky - end else if ($signed(input_exp_s1[i]) < INT_EXP_WIDTH'($signed(1-EXP_BIAS))) begin - // Denormalize underflowing values - final_exp_tmp_s1 = '0; // denormal result - denorm_shamt_s1 = SHAMT_BITS'(1-EXP_BIAS) - SHAMT_BITS'(input_exp_s1[i]); // adjust right shifting - end + if (overflow) begin + denorm_shamt_q = '0; + end else if (underflow) begin + denorm_shamt_q = INT_WIDTH+1; end else begin - if ($signed(input_exp_s1[i]) >= $signed(INT_EXP_WIDTH'(MAX_INT_WIDTH-1) + INT_EXP_WIDTH'(unsigned_s1))) begin - // overflow: when converting to unsigned the range is larger by one - of_before_round_tmp_s1 = 1'b1; - end else if ($signed(input_exp_s1[i]) < INT_EXP_WIDTH'($signed(-1))) begin - // underflow - denorm_shamt_s1 = MAX_INT_WIDTH+1; // all bits go to the sticky - end else begin - // By default right shift mantissa to be an integer - denorm_shamt_s1 = SHAMT_BITS'(MAX_INT_WIDTH-1) - SHAMT_BITS'(input_exp_s1[i]); - end + denorm_shamt_q = denorm_shamt; end end - - assign destination_mant_s1[i] = preshift_mant_s1 >> denorm_shamt_s1; - assign final_exp_s1[i] = final_exp_tmp_s1; - assign of_before_round_s1[i] = of_before_round_tmp_s1; + assign destination_mant_s1[i] = is_itof_s1 ? {input_mant_s1[i], 33'b0} : ({input_mant_s1[i], 33'b0} >> denorm_shamt_q); + assign final_exp_s1[i] = input_exp_s1[i] + INT_EXP_WIDTH'(EXP_BIAS); + assign of_before_round_s1[i] = overflow; end // Pipeline stage2 - wire valid_in_s2; - wire [NUM_LANES-1:0] lane_mask_s2; - wire [TAGW-1:0] tag_in_s2; - wire is_itof_s2; - wire unsigned_s2; - wire [2:0] rnd_mode_s2; + wire valid_in_s2; + wire [NUM_LANES-1:0] lane_mask_s2; + wire [TAGW-1:0] tag_in_s2; + wire is_itof_s2; + wire is_signed_s2; + wire [2:0] rnd_mode_s2; fclass_t [NUM_LANES-1:0] fclass_s2; - wire [NUM_LANES-1:0] mant_is_zero_s2; - wire [NUM_LANES-1:0] input_sign_s2; + wire [NUM_LANES-1:0] mant_is_zero_s2; + wire [NUM_LANES-1:0] input_sign_s2; wire [NUM_LANES-1:0][2*INT_MAN_WIDTH:0] destination_mant_s2; wire [NUM_LANES-1:0][INT_EXP_WIDTH-1:0] final_exp_s2; - wire [NUM_LANES-1:0] of_before_round_s2; + wire [NUM_LANES-1:0] of_before_round_s2; VX_pipe_register #( .DATAW (1 + NUM_LANES + TAGW + 1 + 1 + `INST_FRM_BITS + NUM_LANES * ($bits(fclass_t) + 1 + 1 + (2*INT_MAN_WIDTH+1) + INT_EXP_WIDTH + 1)), @@ -263,37 +228,37 @@ module VX_fpu_cvt import VX_fpu_pkg::*; #( .clk (clk), .reset (reset), .enable (~stall), - .data_in ({valid_in_s1, lane_mask_s1, tag_in_s1, is_itof_s1, unsigned_s1, rnd_mode_s1, fclass_s1, mant_is_zero_s1, input_sign_s1, destination_mant_s1, final_exp_s1, of_before_round_s1}), - .data_out ({valid_in_s2, lane_mask_s2, tag_in_s2, is_itof_s2, unsigned_s2, rnd_mode_s2, fclass_s2, mant_is_zero_s2, input_sign_s2, destination_mant_s2, final_exp_s2, of_before_round_s2}) + .data_in ({valid_in_s1, lane_mask_s1, tag_in_s1, is_itof_s1, is_signed_s1, rnd_mode_s1, fclass_s1, mant_is_zero_s1, input_sign_s1, destination_mant_s1, final_exp_s1, of_before_round_s1}), + .data_out ({valid_in_s2, lane_mask_s2, tag_in_s2, is_itof_s2, is_signed_s2, rnd_mode_s2, fclass_s2, mant_is_zero_s2, input_sign_s2, destination_mant_s2, final_exp_s2, of_before_round_s2}) ); - wire [NUM_LANES-1:0] rounded_sign_s2; - wire [NUM_LANES-1:0][31:0] rounded_abs_s2; // absolute value of result after rounding - wire [NUM_LANES-1:0] int_round_has_sticky_s2; - wire [NUM_LANES-1:0] fp_round_has_sticky_s2; + wire [NUM_LANES-1:0] rounded_sign_s2; + wire [NUM_LANES-1:0][INT_WIDTH-1:0] rounded_abs_s2; // absolute value of result after rounding + wire [NUM_LANES-1:0] f2i_round_has_sticky_s2; + wire [NUM_LANES-1:0] i2f_round_has_sticky_s2; // Rouding and classification for (genvar i = 0; i < NUM_LANES; ++i) begin - wire [MAN_BITS-1:0] final_mant_s2; // mantissa after adjustments - wire [MAX_INT_WIDTH-1:0] final_int_s2; // integer shifted in position - wire [1:0] round_sticky_bits_s2; - wire [31:0] fmt_pre_round_abs_s2; - wire [31:0] pre_round_abs_s2; - wire [1:0] int_round_sticky_bits_s2, fp_round_sticky_bits_s2; + wire [MAN_BITS-1:0] final_mant_s2; // mantissa after adjustments + wire [INT_WIDTH-1:0] final_int_s2; // integer shifted in position + wire [1:0] round_sticky_bits_s2; + wire [INT_WIDTH-1:0] fmt_pre_round_abs_s2; + wire [INT_WIDTH-1:0] pre_round_abs_s2; + wire [1:0] f2i_round_sticky_bits_s2, i2f_round_sticky_bits_s2; // Extract final mantissa and round bit, discard the normal bit (for FP) - assign {final_mant_s2, fp_round_sticky_bits_s2[1]} = destination_mant_s2[i][2*INT_MAN_WIDTH-1 : 2*INT_MAN_WIDTH-1 - (MAN_BITS+1) + 1]; - assign {final_int_s2, int_round_sticky_bits_s2[1]} = destination_mant_s2[i][2*INT_MAN_WIDTH : 2*INT_MAN_WIDTH - (MAX_INT_WIDTH+1) + 1]; + assign {final_mant_s2, i2f_round_sticky_bits_s2[1]} = destination_mant_s2[i][2*INT_MAN_WIDTH-1 : 2*INT_MAN_WIDTH-1 - (MAN_BITS+1) + 1]; + assign {final_int_s2, f2i_round_sticky_bits_s2[1]} = destination_mant_s2[i][2*INT_MAN_WIDTH : 2*INT_MAN_WIDTH - (INT_WIDTH+1) + 1]; // Collapse sticky bits - assign fp_round_sticky_bits_s2[0] = (| destination_mant_s2[i][NUM_FP_STICKY-1:0]); - assign int_round_sticky_bits_s2[0] = (| destination_mant_s2[i][NUM_INT_STICKY-1:0]); - assign fp_round_has_sticky_s2[i] = (| fp_round_sticky_bits_s2); - assign int_round_has_sticky_s2[i] = (| int_round_sticky_bits_s2); + assign i2f_round_sticky_bits_s2[0] = (| destination_mant_s2[i][NUM_FP_STICKY-1:0]); + assign f2i_round_sticky_bits_s2[0] = (| destination_mant_s2[i][NUM_INT_STICKY-1:0]); + assign i2f_round_has_sticky_s2[i] = (| i2f_round_sticky_bits_s2); + assign f2i_round_has_sticky_s2[i] = (| f2i_round_sticky_bits_s2); // select RS bits for destination operation - assign round_sticky_bits_s2 = is_itof_s2 ? fp_round_sticky_bits_s2 : int_round_sticky_bits_s2; + assign round_sticky_bits_s2 = is_itof_s2 ? i2f_round_sticky_bits_s2 : f2i_round_sticky_bits_s2; // Pack exponent and mantissa into proper rounding form assign fmt_pre_round_abs_s2 = {1'b0, final_exp_s2[i][EXP_BITS-1:0], final_mant_s2[MAN_BITS-1:0]}; @@ -322,15 +287,15 @@ module VX_fpu_cvt import VX_fpu_pkg::*; #( wire [NUM_LANES-1:0] lane_mask_s3; wire [TAGW-1:0] tag_in_s3; wire is_itof_s3; - wire unsigned_s3; + wire is_signed_s3; fclass_t [NUM_LANES-1:0] fclass_s3; wire [NUM_LANES-1:0] mant_is_zero_s3; wire [NUM_LANES-1:0] input_sign_s3; wire [NUM_LANES-1:0] rounded_sign_s3; - wire [NUM_LANES-1:0][31:0] rounded_abs_s3; + wire [NUM_LANES-1:0][INT_WIDTH-1:0] rounded_abs_s3; wire [NUM_LANES-1:0] of_before_round_s3; - wire [NUM_LANES-1:0] int_round_has_sticky_s3; - wire [NUM_LANES-1:0] fp_round_has_sticky_s3; + wire [NUM_LANES-1:0] f2i_round_has_sticky_s3; + wire [NUM_LANES-1:0] i2f_round_has_sticky_s3; VX_pipe_register #( .DATAW (1 + NUM_LANES + TAGW + 1 + 1 + NUM_LANES * ($bits(fclass_t) + 1 + 1 + 32 + 1 + 1 + 1 + 1)), @@ -339,105 +304,71 @@ module VX_fpu_cvt import VX_fpu_pkg::*; #( .clk (clk), .reset (reset), .enable (~stall), - .data_in ({valid_in_s2, lane_mask_s2, tag_in_s2, is_itof_s2, unsigned_s2, fclass_s2, mant_is_zero_s2, input_sign_s2, rounded_abs_s2, rounded_sign_s2, of_before_round_s2, int_round_has_sticky_s2, fp_round_has_sticky_s2}), - .data_out ({valid_in_s3, lane_mask_s3, tag_in_s3, is_itof_s3, unsigned_s3, fclass_s3, mant_is_zero_s3, input_sign_s3, rounded_abs_s3, rounded_sign_s3, of_before_round_s3, int_round_has_sticky_s3, fp_round_has_sticky_s3}) + .data_in ({valid_in_s2, lane_mask_s2, tag_in_s2, is_itof_s2, is_signed_s2, fclass_s2, mant_is_zero_s2, input_sign_s2, rounded_abs_s2, rounded_sign_s2, of_before_round_s2, f2i_round_has_sticky_s2, i2f_round_has_sticky_s2}), + .data_out ({valid_in_s3, lane_mask_s3, tag_in_s3, is_itof_s3, is_signed_s3, fclass_s3, mant_is_zero_s3, input_sign_s3, rounded_abs_s3, rounded_sign_s3, of_before_round_s3, f2i_round_has_sticky_s3, i2f_round_has_sticky_s3}) ); - wire [NUM_LANES-1:0] of_after_round_s3; - wire [NUM_LANES-1:0] uf_after_round_s3; - wire [NUM_LANES-1:0][31:0] fmt_result_s3; - wire [NUM_LANES-1:0][31:0] rounded_int_res_s3; // after possible inversion + wire [NUM_LANES-1:0][INT_WIDTH-1:0] fmt_result_s3; + wire [NUM_LANES-1:0][INT_WIDTH-1:0] rounded_int_res_s3; // after possible inversion wire [NUM_LANES-1:0] rounded_int_res_zero_s3; // after rounding for (genvar i = 0; i < NUM_LANES; ++i) begin // Assemble regular result, nan box short ones. Int zeroes need to be detected - assign fmt_result_s3[i] = (is_itof_s3 & mant_is_zero_s3[i]) ? 0 : {rounded_sign_s3[i], rounded_abs_s3[i][EXP_BITS+MAN_BITS-1:0]}; - - // Classification after rounding select by destination format - assign uf_after_round_s3[i] = (rounded_abs_s3[i][EXP_BITS+MAN_BITS-1:MAN_BITS] == 0); // denormal - assign of_after_round_s3[i] = (rounded_abs_s3[i][EXP_BITS+MAN_BITS-1:MAN_BITS] == ~0); // inf exp. + assign fmt_result_s3[i] = mant_is_zero_s3[i] ? 0 : {rounded_sign_s3[i], rounded_abs_s3[i][EXP_BITS+MAN_BITS-1:0]}; // Negative integer result needs to be brought into two's complement assign rounded_int_res_s3[i] = rounded_sign_s3[i] ? (-rounded_abs_s3[i]) : rounded_abs_s3[i]; assign rounded_int_res_zero_s3[i] = (rounded_int_res_s3[i] == 0); end - // FP Special case handling + // F2I Special case handling - wire [NUM_LANES-1:0][31:0] fp_special_result_s3; - fflags_t [NUM_LANES-1:0] fp_special_status_s3; - wire [NUM_LANES-1:0] fp_result_is_special_s3; - - for (genvar i = 0; i < NUM_LANES; ++i) begin - // Detect special case from source format, I2F casts don't produce a special result - assign fp_result_is_special_s3[i] = ~is_itof_s3 & (fclass_s3[i].is_zero | fclass_s3[i].is_nan); - - // Signalling input NaNs raise invalid flag, otherwise no flags set - assign fp_special_status_s3[i] = fclass_s3[i].is_signaling ? {1'b1, 4'h0} : 5'h0; // invalid operation - - // Assemble result according to destination format - assign fp_special_result_s3[i] = fclass_s3[i].is_zero ? (32'(input_sign_s3) << 31) // signed zero - : {1'b0, QNAN_EXPONENT, QNAN_MANTISSA}; // qNaN - end - - // INT Special case handling - - reg [NUM_LANES-1:0][31:0] int_special_result_s3; - fflags_t [NUM_LANES-1:0] int_special_status_s3; - wire [NUM_LANES-1:0] int_result_is_special_s3; + reg [NUM_LANES-1:0][INT_WIDTH-1:0] f2i_special_result_s3; + fflags_t [NUM_LANES-1:0] f2i_special_status_s3; + wire [NUM_LANES-1:0] f2i_result_is_special_s3; for (genvar i = 0; i < NUM_LANES; ++i) begin // Assemble result according to destination format always @(*) begin if (input_sign_s3[i] && !fclass_s3[i].is_nan) begin - int_special_result_s3[i][30:0] = '0; // alone yields 2**(31)-1 - int_special_result_s3[i][31] = ~unsigned_s3; // for unsigned casts yields 2**31 + f2i_special_result_s3[i][INT_WIDTH-2:0] = '0; // alone yields 2**(31)-1 + f2i_special_result_s3[i][INT_WIDTH-1] = is_signed_s3; // for unsigned casts yields 2**31 end else begin - int_special_result_s3[i][30:0] = 2**(31) - 1; // alone yields 2**(31)-1 - int_special_result_s3[i][31] = unsigned_s3; // for unsigned casts yields 2**31 + f2i_special_result_s3[i][INT_WIDTH-2:0] = 2**(INT_WIDTH-1) - 1; // alone yields 2**(31)-1 + f2i_special_result_s3[i][INT_WIDTH-1] = ~is_signed_s3; // for unsigned casts yields 2**31 end end // Detect special case from source format (inf, nan, overflow, nan-boxing or negative unsigned) - assign int_result_is_special_s3[i] = fclass_s3[i].is_nan + assign f2i_result_is_special_s3[i] = fclass_s3[i].is_nan | fclass_s3[i].is_inf | of_before_round_s3[i] - | (input_sign_s3[i] & unsigned_s3 & ~rounded_int_res_zero_s3[i]); + | (input_sign_s3[i] & ~is_signed_s3 & ~rounded_int_res_zero_s3[i]); // All integer special cases are invalid - assign int_special_status_s3[i] = {1'b1, 4'h0}; + assign f2i_special_status_s3[i] = {1'b1, 4'h0}; end // Result selection and Output handshake fflags_t [NUM_LANES-1:0] tmp_fflags_s3; - wire [NUM_LANES-1:0][31:0] tmp_result_s3; + wire [NUM_LANES-1:0][INT_WIDTH-1:0] tmp_result_s3; - for (genvar i = 0; i < NUM_LANES; ++i) begin - fflags_t fp_regular_status_s3, int_regular_status_s3; - fflags_t fp_status_s3, int_status_s3; - wire [31:0] fp_result_s3, int_result_s3; + for (genvar i = 0; i < NUM_LANES; ++i) begin + fflags_t i2f_regular_status_s3, f2i_regular_status_s3; + fflags_t i2f_status_s3, f2i_status_s3; - wire inexact_s3 = is_itof_s3 ? fp_round_has_sticky_s3[i] // overflow is invalid in i2f; - : (fp_round_has_sticky_s3[i] || (~fclass_s3[i].is_inf && (of_before_round_s3[i] || of_after_round_s3[i]))); - - assign fp_regular_status_s3.NV = is_itof_s3 & (of_before_round_s3[i] | of_after_round_s3[i]); // overflow is invalid for I2F casts - assign fp_regular_status_s3.DZ = 1'b0; // no divisions - assign fp_regular_status_s3.OF = ~is_itof_s3 & (~fclass_s3[i].is_inf & (of_before_round_s3[i] | of_after_round_s3[i])); // inf casts no OF - assign fp_regular_status_s3.UF = uf_after_round_s3[i] & inexact_s3; - assign fp_regular_status_s3.NX = inexact_s3; + assign i2f_regular_status_s3 = {4'h0, i2f_round_has_sticky_s3[i]}; + assign f2i_regular_status_s3 = {4'h0, f2i_round_has_sticky_s3[i]}; - assign int_regular_status_s3 = int_round_has_sticky_s3[i] ? {4'h0, 1'b1} : 5'h0; + assign i2f_status_s3 = i2f_regular_status_s3; + assign f2i_status_s3 = f2i_result_is_special_s3[i] ? f2i_special_status_s3[i] : f2i_regular_status_s3; - assign fp_result_s3 = fp_result_is_special_s3[i] ? fp_special_result_s3[i] : fmt_result_s3[i]; - assign int_result_s3 = int_result_is_special_s3[i] ? int_special_result_s3[i] : rounded_int_res_s3[i]; + wire [INT_WIDTH-1:0] i2f_result_s3 = fmt_result_s3[i]; + wire [INT_WIDTH-1:0] f2i_result_s3 = f2i_result_is_special_s3[i] ? f2i_special_result_s3[i] : rounded_int_res_s3[i]; - assign fp_status_s3 = fp_result_is_special_s3[i] ? fp_special_status_s3[i] : fp_regular_status_s3; - assign int_status_s3 = int_result_is_special_s3[i] ? int_special_status_s3[i] : int_regular_status_s3; - - // Select output depending on special case detection - assign tmp_result_s3[i] = is_itof_s3 ? fp_result_s3 : int_result_s3; - assign tmp_fflags_s3[i] = is_itof_s3 ? fp_status_s3 : int_status_s3; + assign tmp_result_s3[i] = is_itof_s3 ? i2f_result_s3 : f2i_result_s3; + assign tmp_fflags_s3[i] = is_itof_s3 ? i2f_status_s3 : f2i_status_s3; end assign stall = ~ready_out && valid_out; @@ -457,7 +388,6 @@ module VX_fpu_cvt import VX_fpu_pkg::*; #( ); assign ready_in = ~stall; - assign has_fflags = 1'b1; endmodule diff --git a/hw/rtl/fpu/VX_fpu_define.vh b/hw/rtl/fpu/VX_fpu_define.vh index a72914ef..596db920 100644 --- a/hw/rtl/fpu/VX_fpu_define.vh +++ b/hw/rtl/fpu/VX_fpu_define.vh @@ -16,7 +16,7 @@ `include "VX_define.vh" -`ifndef SYNTHESIS +`ifdef SV_DPI `include "float_dpi.vh" `endif diff --git a/hw/rtl/fpu/VX_fpu_rounding.sv b/hw/rtl/fpu/VX_fpu_rounding.sv index 5168fada..877b4eb6 100644 --- a/hw/rtl/fpu/VX_fpu_rounding.sv +++ b/hw/rtl/fpu/VX_fpu_rounding.sv @@ -54,7 +54,6 @@ module VX_fpu_rounding #( 2'b01: round_up = 1'b0; // < ulp/2 away, round down 2'b10: round_up = abs_value_i[0]; // = ulp/2 away, round towards even result 2'b11: round_up = 1'b1; // > ulp/2 away, round up - default: round_up = 1'bx; endcase `INST_FRM_RTZ: round_up = 1'b0; // always round down `INST_FRM_RDN: round_up = (| round_sticky_bits_i) & sign_i; // to 0 if +, away if - diff --git a/hw/rtl/interfaces/VX_pipeline_perf_if.sv b/hw/rtl/interfaces/VX_pipeline_perf_if.sv index b6123b7f..7d421875 100644 --- a/hw/rtl/interfaces/VX_pipeline_perf_if.sv +++ b/hw/rtl/interfaces/VX_pipeline_perf_if.sv @@ -14,26 +14,38 @@ `include "VX_define.vh" interface VX_pipeline_perf_if (); - wire [`PERF_CTR_BITS-1:0] ibf_stalls; - wire [`PERF_CTR_BITS-1:0] scb_stalls; - wire [`PERF_CTR_BITS-1:0] dsp_stalls [`NUM_EX_UNITS]; + wire [`PERF_CTR_BITS-1:0] sched_idles; + wire [`PERF_CTR_BITS-1:0] sched_stalls; + wire [`PERF_CTR_BITS-1:0] ibf_stalls; + wire [`PERF_CTR_BITS-1:0] scb_stalls; + wire [`PERF_CTR_BITS-1:0] units_uses [`NUM_EX_UNITS]; + wire [`PERF_CTR_BITS-1:0] sfu_uses [`NUM_SFU_UNITS]; - wire [`PERF_CTR_BITS-1:0] ifetches; - wire [`PERF_CTR_BITS-1:0] loads; - wire [`PERF_CTR_BITS-1:0] stores; - wire [`PERF_CTR_BITS-1:0] ifetch_latency; - wire [`PERF_CTR_BITS-1:0] load_latency; + wire [`PERF_CTR_BITS-1:0] ifetches; + wire [`PERF_CTR_BITS-1:0] loads; + wire [`PERF_CTR_BITS-1:0] stores; + wire [`PERF_CTR_BITS-1:0] ifetch_latency; + wire [`PERF_CTR_BITS-1:0] load_latency; + + modport schedule ( + output sched_idles, + output sched_stalls + ); modport issue ( output ibf_stalls, output scb_stalls, - output dsp_stalls - ); + output units_uses, + output sfu_uses + ); modport slave ( + input sched_idles, + input sched_stalls, input ibf_stalls, input scb_stalls, - input dsp_stalls, + input units_uses, + input sfu_uses, input ifetches, input loads, input stores, diff --git a/hw/rtl/libs/VX_avs_adapter.sv b/hw/rtl/libs/VX_avs_adapter.sv index 779eb45e..4ea53757 100644 --- a/hw/rtl/libs/VX_avs_adapter.sv +++ b/hw/rtl/libs/VX_avs_adapter.sv @@ -21,8 +21,8 @@ module VX_avs_adapter #( parameter NUM_BANKS = 1, parameter TAG_WIDTH = 1, parameter RD_QUEUE_SIZE = 1, - parameter OUT_REG_REQ = 0, - parameter OUT_REG_RSP = 0 + parameter OUT_REG_REQ = 0, + parameter OUT_REG_RSP = 0 ) ( input wire clk, input wire reset, diff --git a/hw/rtl/libs/VX_axi_adapter.sv b/hw/rtl/libs/VX_axi_adapter.sv index 967c3af1..c5919b7a 100644 --- a/hw/rtl/libs/VX_axi_adapter.sv +++ b/hw/rtl/libs/VX_axi_adapter.sv @@ -20,7 +20,7 @@ module VX_axi_adapter #( parameter TAG_WIDTH = 8, parameter NUM_BANKS = 1, parameter AVS_ADDR_WIDTH = (ADDR_WIDTH - `CLOG2(DATA_WIDTH/8)), - parameter OUT_REG_RSP = 0 + parameter OUT_REG_RSP = 0 ) ( input wire clk, input wire reset, diff --git a/hw/rtl/libs/VX_fifo_queue.sv b/hw/rtl/libs/VX_fifo_queue.sv index 78a2785c..1eda9fff 100644 --- a/hw/rtl/libs/VX_fifo_queue.sv +++ b/hw/rtl/libs/VX_fifo_queue.sv @@ -201,9 +201,7 @@ module VX_fifo_queue #( rd_ptr_r <= '0; rd_ptr_n_r <= 1; end else begin - if (push) begin - wr_ptr_r <= wr_ptr_r + ADDRW'(1); - end + wr_ptr_r <= wr_ptr_r + ADDRW'(push); if (pop) begin rd_ptr_r <= rd_ptr_n_r; if (DEPTH > 2) begin diff --git a/hw/rtl/libs/VX_mem_adapter.sv b/hw/rtl/libs/VX_mem_adapter.sv index 19d65240..ed983836 100644 --- a/hw/rtl/libs/VX_mem_adapter.sv +++ b/hw/rtl/libs/VX_mem_adapter.sv @@ -21,8 +21,8 @@ module VX_mem_adapter #( parameter DST_ADDR_WIDTH = 1, parameter SRC_TAG_WIDTH = 1, parameter DST_TAG_WIDTH = 1, - parameter OUT_REG_REQ = 0, - parameter OUT_REG_RSP = 0 + parameter OUT_REG_REQ = 0, + parameter OUT_REG_RSP = 0 ) ( input wire clk, input wire reset, diff --git a/hw/rtl/libs/VX_stream_arb.sv b/hw/rtl/libs/VX_stream_arb.sv index 58da0b25..a81be3ef 100644 --- a/hw/rtl/libs/VX_stream_arb.sv +++ b/hw/rtl/libs/VX_stream_arb.sv @@ -21,7 +21,7 @@ module VX_stream_arb #( parameter `STRING ARBITER = "P", parameter LOCK_ENABLE = 1, parameter MAX_FANOUT = `MAX_FANOUT, - parameter OUT_REG = 0 , + parameter OUT_REG = 0 , parameter NUM_REQS = (NUM_INPUTS + NUM_OUTPUTS - 1) / NUM_OUTPUTS, parameter LOG_NUM_REQS = `CLOG2(NUM_REQS), parameter NUM_REQS_W = `UP(LOG_NUM_REQS) diff --git a/hw/rtl/libs/VX_stream_xbar.sv b/hw/rtl/libs/VX_stream_xbar.sv index db92cfd0..2a8e4bb4 100644 --- a/hw/rtl/libs/VX_stream_xbar.sv +++ b/hw/rtl/libs/VX_stream_xbar.sv @@ -173,25 +173,27 @@ module VX_stream_xbar #( end // compute inputs collision - // we have a collision when there exists a valid transfer with mutiple input candicates - // we caount the unique duplicates each cycle. + // we have a collision when there exists a valid transfer with multiple input candicates + // we count the unique duplicates each cycle. + reg [NUM_INPUTS-1:0] per_cycle_collision, per_cycle_collision_r; + wire [`CLOG2(NUM_INPUTS+1)-1:0] collision_count; reg [PERF_CTR_BITS-1:0] collisions_r; - reg [NUM_INPUTS-1:0] per_cycle_collision; always @(*) begin per_cycle_collision = 0; for (integer i = 0; i < NUM_INPUTS; ++i) begin for (integer j = 1; j < (NUM_INPUTS-i); ++j) begin - if (valid_in[i] && valid_in[j+i] && sel_in[i] == sel_in[j+i]) begin - per_cycle_collision[i] |= ready_in[i] | ready_in[j+i]; - end + per_cycle_collision[i] |= valid_in[i] + && valid_in[j+i] + && (sel_in[i] == sel_in[j+i]) + && (ready_in[i] | ready_in[j+i]); end end end - - wire [`CLOG2(NUM_INPUTS+1)-1:0] collision_count; - `POP_COUNT(collision_count, per_cycle_collision); + + `BUFFER(per_cycle_collision_r, per_cycle_collision); + `POP_COUNT(collision_count, per_cycle_collision_r); always @(posedge clk) begin if (reset) begin diff --git a/hw/rtl/mem/VX_gbar_arb.sv b/hw/rtl/mem/VX_gbar_arb.sv index 6aa93510..a4cc07c3 100644 --- a/hw/rtl/mem/VX_gbar_arb.sv +++ b/hw/rtl/mem/VX_gbar_arb.sv @@ -15,7 +15,7 @@ module VX_gbar_arb #( parameter NUM_REQS = 1, - parameter OUT_REG = 0, + parameter OUT_REG = 0, parameter `STRING ARBITER = "R" ) ( input wire clk, diff --git a/hw/rtl/mem/VX_mem_arb.sv b/hw/rtl/mem/VX_mem_arb.sv index 939dd6ba..2588a9ea 100644 --- a/hw/rtl/mem/VX_mem_arb.sv +++ b/hw/rtl/mem/VX_mem_arb.sv @@ -21,8 +21,8 @@ module VX_mem_arb #( parameter ADDR_WIDTH = (MEM_ADDR_WIDTH-`CLOG2(DATA_SIZE)), parameter TAG_WIDTH = 1, parameter TAG_SEL_IDX = 0, - parameter OUT_REG_REQ = 0, - parameter OUT_REG_RSP = 0, + parameter OUT_REG_REQ = 0, + parameter OUT_REG_RSP = 0, parameter `STRING ARBITER = "R" ) ( input wire clk, diff --git a/hw/rtl/mem/VX_shared_mem.sv b/hw/rtl/mem/VX_shared_mem.sv index a44c68a8..97082468 100644 --- a/hw/rtl/mem/VX_shared_mem.sv +++ b/hw/rtl/mem/VX_shared_mem.sv @@ -233,10 +233,12 @@ module VX_shared_mem import VX_gpu_pkg::*; #( wire [`CLOG2(NUM_REQS+1)-1:0] perf_writes_per_cycle; wire [`CLOG2(NUM_REQS+1)-1:0] perf_crsp_stall_per_cycle; - wire [NUM_REQS-1:0] perf_reads_per_req = req_valid & req_ready & ~req_rw; - wire [NUM_REQS-1:0] perf_writes_per_req = req_valid & req_ready & req_rw; + wire [NUM_REQS-1:0] perf_reads_per_req, perf_writes_per_req; wire [NUM_REQS-1:0] perf_crsp_stall_per_req = rsp_valid & ~rsp_ready; + `BUFFER(perf_reads_per_req, req_valid & req_ready & ~req_rw); + `BUFFER(perf_writes_per_req, req_valid & req_ready & req_rw); + `POP_COUNT(perf_reads_per_cycle, perf_reads_per_req); `POP_COUNT(perf_writes_per_cycle, perf_writes_per_req); `POP_COUNT(perf_crsp_stall_per_cycle, perf_crsp_stall_per_req); diff --git a/hw/rtl/mem/VX_smem_switch.sv b/hw/rtl/mem/VX_smem_switch.sv index 7dc410a9..5fb92915 100644 --- a/hw/rtl/mem/VX_smem_switch.sv +++ b/hw/rtl/mem/VX_smem_switch.sv @@ -19,8 +19,8 @@ module VX_smem_switch #( parameter TAG_WIDTH = 1, parameter MEM_ADDR_WIDTH = `MEM_ADDR_WIDTH, parameter TAG_SEL_IDX = 0, - parameter OUT_REG_REQ = 0, - parameter OUT_REG_RSP = 0, + parameter OUT_REG_REQ = 0, + parameter OUT_REG_RSP = 0, parameter `STRING ARBITER = "R" ) ( input wire clk, diff --git a/hw/syn/altera/NOTEBOOK b/hw/syn/altera/NOTEBOOK index 9b871546..7929e593 100644 --- a/hw/syn/altera/NOTEBOOK +++ b/hw/syn/altera/NOTEBOOK @@ -56,17 +56,17 @@ TARGET=asesim make -C runtime/opae PREFIX=build_base CONFIGS="-DEXT_F_DISABLE -DL1_DISABLE -DSM_DISABLE -DNUM_WARPS=2 -DNUM_THREADS=2" TARGET=asesim make # ASE test runs -./run_ase.sh build_base_arria10_asesim_1c/synth ../../../../tests/regression/basic/basic -n1 -t0 -./run_ase.sh build_base_arria10_asesim_1c/synth ../../../../tests/regression/basic/basic -n1 -t1 -./run_ase.sh build_base_arria10_asesim_1c/synth ../../../../tests/regression/basic/basic -n16 -./run_ase.sh build_base_arria10_asesim_1c/synth ../../../../tests/regression/demo/demo -n16 -./run_ase.sh build_base_arria10_asesim_1c/synth ../../../../tests/regression/dogfood/dogfood -n16 -./run_ase.sh build_base_arria10_asesim_1c/synth ../../../../tests/opencl/vecadd/vecadd -./run_ase.sh build_base_arria10_asesim_1c/synth ../../../../tests/opencl/sgemm/sgemm -n4 +./run_ase.sh build_base_arria10_asesim_1c ../../../../tests/regression/basic/basic -n1 -t0 +./run_ase.sh build_base_arria10_asesim_1c ../../../../tests/regression/basic/basic -n1 -t1 +./run_ase.sh build_base_arria10_asesim_1c ../../../../tests/regression/basic/basic -n16 +./run_ase.sh build_base_arria10_asesim_1c ../../../../tests/regression/demo/demo -n16 +./run_ase.sh build_base_arria10_asesim_1c ../../../../tests/regression/dogfood/dogfood -n16 +./run_ase.sh build_base_arria10_asesim_1c ../../../../tests/opencl/vecadd/vecadd +./run_ase.sh build_base_arria10_asesim_1c ../../../../tests/opencl/sgemm/sgemm -n4 # modify "vsim_run.tcl" to dump VCD trace vcd file trace.vcd -vcd add -r /*/Vortex/hw/rtl/* +vcd add -r /*/afu/* run -all # compress FPGA output files diff --git a/hw/syn/altera/opae/run_ase.sh b/hw/syn/altera/opae/run_ase.sh index ba30d209..16c92f45 100755 --- a/hw/syn/altera/opae/run_ase.sh +++ b/hw/syn/altera/opae/run_ase.sh @@ -15,27 +15,27 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -BUILD_DIR=$1 +BUILD_DIR=$(realpath $1) PROGRAM=$(basename "$2") PROGRAM_DIR=`dirname $2` +POCL_RT_PATH=$TOOLDIR/pocl/runtime VORTEX_RT_PATH=$SCRIPT_DIR/../../../../runtime # Export ASE_WORKDIR variable -export ASE_WORKDIR=$SCRIPT_DIR/$BUILD_DIR/work - -shift 2 +export ASE_WORKDIR=$BUILD_DIR/synth/work # cleanup incomplete runs rm -f $ASE_WORKDIR/.app_lock.pid rm -f $ASE_WORKDIR/.ase_ready.pid -rm -f $SCRIPT_DIR/$BUILD_DIR/nohup.out +rm -f $BUILD_DIR/synth/nohup.out -# Start Simulator in background -pushd $SCRIPT_DIR/$BUILD_DIR -echo " [DBG] starting ASE simnulator (stdout saved to '$SCRIPT_DIR/$BUILD_DIR/nohup.out')" -nohup make sim & +# Start Simulator in background (capture processs group pid) +pushd $BUILD_DIR/synth +echo " [DBG] starting ASE simnulator (stdout saved to '$BUILD_DIR/synth/nohup.out')" +setsid make sim &> /dev/null & +SIM_PID=$! popd # Wait for simulator readiness @@ -47,6 +47,11 @@ done # run application pushd $PROGRAM_DIR +shift 2 echo " [DBG] running ./$PROGRAM $*" ASE_LOG=0 LD_LIBRARY_PATH=$POCL_RT_PATH/lib:$VORTEX_RT_PATH/opae:$LD_LIBRARY_PATH ./$PROGRAM $* popd + +# stop the simulator (kill process group) +kill -- -$(ps -o pgid= $SIM_PID | grep -o '[0-9]*') +wait $SIM_PID 2> /dev/null \ No newline at end of file diff --git a/hw/syn/altera/quartus/cache/Makefile b/hw/syn/altera/quartus/cache/Makefile index 258dc91a..f96a7614 100755 --- a/hw/syn/altera/quartus/cache/Makefile +++ b/hw/syn/altera/quartus/cache/Makefile @@ -1,6 +1,6 @@ -PROJECT = VX_cache_cluster_top +PROJECT = VX_cache_top TOP_LEVEL_ENTITY = $(PROJECT) -SRC_FILE = VX_cache_cluster.sv +SRC_FILE = $(PROJECT).sv include ../../common.mk diff --git a/hw/syn/altera/quartus/core/Makefile b/hw/syn/altera/quartus/core/Makefile index f1dc07f3..eeeaa523 100644 --- a/hw/syn/altera/quartus/core/Makefile +++ b/hw/syn/altera/quartus/core/Makefile @@ -1,6 +1,6 @@ PROJECT = VX_core_top TOP_LEVEL_ENTITY = $(PROJECT) -SRC_FILE = VX_core.sv +SRC_FILE = $(PROJECT).sv include ../../common.mk diff --git a/hw/syn/xilinx/test/kernel/Makefile b/hw/syn/xilinx/test/kernel/Makefile index 55c21aa2..11457ab4 100644 --- a/hw/syn/xilinx/test/kernel/Makefile +++ b/hw/syn/xilinx/test/kernel/Makefile @@ -1,10 +1,11 @@ XLEN ?= 32 +TOOLDIR ?= /opt ifeq ($(XLEN),64) -RISCV_TOOLCHAIN_PATH ?= /opt/riscv64-gnu-toolchain +RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv64-gnu-toolchain CFLAGS += -march=rv64imafd -mabi=lp64d else -RISCV_TOOLCHAIN_PATH ?= /opt/riscv-gnu-toolchain +RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv-gnu-toolchain CFLAGS += -march=rv32imaf -mabi=ilp32f endif diff --git a/hw/unittest/cache/Makefile b/hw/unittest/cache/Makefile index e3737c3c..0d5da094 100644 --- a/hw/unittest/cache/Makefile +++ b/hw/unittest/cache/Makefile @@ -33,7 +33,7 @@ VL_FLAGS = --exe VL_FLAGS += --language 1800-2009 --assert -Wall -Wpedantic VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique -VL_FLAGS += -DSIMULATION +VL_FLAGS += -DSIMULATION -DSV_DPI VL_FLAGS += $(CONFIGS) VL_FLAGS += $(PARAMS) VL_FLAGS += $(RTL_INCLUDE) diff --git a/hw/unittest/generic_queue/Makefile b/hw/unittest/generic_queue/Makefile index 9ec4baa1..c25bc006 100644 --- a/hw/unittest/generic_queue/Makefile +++ b/hw/unittest/generic_queue/Makefile @@ -27,7 +27,7 @@ VL_FLAGS = --exe VL_FLAGS += --language 1800-2009 --assert -Wall -Wpedantic VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique -VL_FLAGS += -DSIMULATION +VL_FLAGS += -DSIMULATION -DSV_DPI VL_FLAGS += $(CONFIGS) VL_FLAGS += $(PARAMS) VL_FLAGS += $(RTL_INCLUDE) diff --git a/hw/unittest/mem_streamer/Makefile b/hw/unittest/mem_streamer/Makefile index 9ff5e81c..aa4b517a 100644 --- a/hw/unittest/mem_streamer/Makefile +++ b/hw/unittest/mem_streamer/Makefile @@ -27,7 +27,7 @@ VL_FLAGS = --exe VL_FLAGS += --language 1800-2009 --assert -Wall -Wpedantic VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique -VL_FLAGS += -DSIMULATION +VL_FLAGS += -DSIMULATION -DSV_DPI VL_FLAGS += $(CONFIGS) VL_FLAGS += $(PARAMS) VL_FLAGS += $(RTL_INCLUDE) diff --git a/hw/unittest/top_modules/Makefile b/hw/unittest/top_modules/Makefile index 2d0319e7..72a403c5 100644 --- a/hw/unittest/top_modules/Makefile +++ b/hw/unittest/top_modules/Makefile @@ -25,7 +25,7 @@ VL_FLAGS = --exe VL_FLAGS += --language 1800-2009 --assert -Wall -Wpedantic VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique -VL_FLAGS += -DSIMULATION +VL_FLAGS += -DSIMULATION -DSV_DPI VL_FLAGS += $(CONFIGS) VL_FLAGS += $(PARAMS) VL_FLAGS += $(RTL_INCLUDE) @@ -56,7 +56,6 @@ PROJECT = top_modules all: build build: $(SRCS) - verilator --build $(VL_FLAGS) --cc VX_cache_cluster_top --top-module VX_cache_cluster_top $^ -CFLAGS '$(CXXFLAGS)' verilator --build $(VL_FLAGS) --cc VX_cache_top --top-module VX_cache_top $^ -CFLAGS '$(CXXFLAGS)' verilator --build $(VL_FLAGS) --cc VX_core_top --top-module VX_core_top $^ -CFLAGS '$(CXXFLAGS)' diff --git a/kernel/Makefile b/kernel/Makefile index e4c975dc..07b8c97b 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -1,17 +1,18 @@ XLEN ?= 32 +TOOLDIR ?= /opt ifeq ($(XLEN),64) -RISCV_TOOLCHAIN_PATH ?= /opt/riscv64-gnu-toolchain +RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv64-gnu-toolchain CFLAGS += -march=rv64imafd -mabi=lp64d else -RISCV_TOOLCHAIN_PATH ?= /opt/riscv-gnu-toolchain +RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv-gnu-toolchain CFLAGS += -march=rv32imaf -mabi=ilp32f endif RISCV_PREFIX ?= riscv$(XLEN)-unknown-elf RISCV_SYSROOT ?= $(RISCV_TOOLCHAIN_PATH)/$(RISCV_PREFIX) -LLVM_VORTEX ?= /opt/llvm-vortex +LLVM_VORTEX ?= $(TOOLDIR)/llvm-vortex LLVM_CFLAGS += --sysroot=$(RISCV_SYSROOT) LLVM_CFLAGS += --gcc-toolchain=$(RISCV_TOOLCHAIN_PATH) diff --git a/kernel/src/vx_spawn.c b/kernel/src/vx_spawn.c index 14773707..fd8258e1 100644 --- a/kernel/src/vx_spawn.c +++ b/kernel/src/vx_spawn.c @@ -51,9 +51,8 @@ inline char is_log2(int x) { return ((x & (x-1)) == 0); } -inline int fast_log2(int x) { - float f = x; - return (*(int*)(&f)>>23) - 127; +inline int log2_fast(int x) { + return 31 - __builtin_clz (x); } static void __attribute__ ((noinline)) spawn_tasks_all_stub() { @@ -286,8 +285,8 @@ void vx_spawn_kernel(context_t * ctx, vx_spawn_kernel_cb callback, void * arg) { // fast path handling char isXYpow2 = is_log2(XY); - char log2XY = fast_log2(XY); - char log2X = fast_log2(X); + char log2XY = log2_fast(XY); + char log2X = log2_fast(X); wspawn_kernel_args_t wspawn_args = { ctx, callback, arg, core_id * tasks_per_core, fW, rW, isXYpow2, log2XY, log2X diff --git a/runtime/common/utils.cpp b/runtime/common/utils.cpp index 72c2b80c..6d601555 100644 --- a/runtime/common/utils.cpp +++ b/runtime/common/utils.cpp @@ -175,8 +175,9 @@ static uint64_t get_csr_64(const void* ptr, int addr) { extern int vx_dump_perf(vx_device_h hdevice, FILE* stream) { int ret = 0; - uint64_t instrs = 0; - uint64_t cycles = 0; + uint64_t total_instrs = 0; + uint64_t total_cycles = 0; + uint64_t max_cycles = 0; #ifdef PERF_ENABLE @@ -186,27 +187,29 @@ extern int vx_dump_perf(vx_device_h hdevice, FILE* stream) { return int((1.0 - (double(part) / double(total))) * 100); }; - auto caclAvgLatency = [&](uint64_t sum, uint64_t requests)->int { - if (requests == 0) + auto caclAverage = [&](uint64_t part, uint64_t total)->double { + if (total == 0) return 0; - return int(double(sum) / double(requests)); + return double(part) / double(total); }; - auto calcUtilization = [&](uint64_t count, uint64_t stalls)->int { - if (count == 0) - return 0; - return int((double(count) / double(count + stalls)) * 100); + auto calcAvgPercent = [&](uint64_t part, uint64_t total)->int { + return int(caclAverage(part, total) * 100); }; auto perf_class = gAutoPerfDump.get_perf_class(); // PERF: pipeline stalls + uint64_t sched_idles = 0; + uint64_t sched_stalls = 0; uint64_t ibuffer_stalls = 0; - uint64_t scoreboard_stalls = 0; - uint64_t lsu_stalls = 0; - uint64_t fpu_stalls = 0; - uint64_t alu_stalls = 0; - uint64_t sfu_stalls = 0; + uint64_t scrb_stalls = 0; + uint64_t scrb_alu = 0; + uint64_t scrb_fpu = 0; + uint64_t scrb_lsu = 0; + uint64_t scrb_sfu = 0; + uint64_t scrb_wctl = 0; + uint64_t scrb_csrs = 0; uint64_t ifetches = 0; uint64_t loads = 0; uint64_t stores = 0; @@ -251,76 +254,126 @@ extern int vx_dump_perf(vx_device_h hdevice, FILE* stream) { #endif std::vector staging_buf(64* sizeof(uint32_t)); - - for (unsigned core_id = 0; core_id < num_cores; ++core_id) { + + for (unsigned core_id = 0; core_id < num_cores; ++core_id) { uint64_t mpm_mem_addr = IO_CSR_ADDR + core_id * staging_buf.size(); ret = vx_copy_from_dev(hdevice, staging_buf.data(), mpm_mem_addr, staging_buf.size()); if (ret != 0) return ret; + uint64_t cycles_per_core = get_csr_64(staging_buf.data(), VX_CSR_MCYCLE); + uint64_t instrs_per_core = get_csr_64(staging_buf.data(), VX_CSR_MINSTRET); + #ifdef PERF_ENABLE switch (perf_class) { case VX_DCR_MPM_CLASS_CORE: { // PERF: pipeline - // ibuffer_stall - uint64_t ibuffer_stalls_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_IBUF_ST); - if (num_cores > 1) fprintf(stream, "PERF: core%d: ibuffer stalls=%ld\n", core_id, ibuffer_stalls_per_core); - ibuffer_stalls += ibuffer_stalls_per_core; - // scoreboard_stall - uint64_t scoreboard_stalls_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SCRB_ST); - if (num_cores > 1) fprintf(stream, "PERF: core%d: scoreboard stalls=%ld\n", core_id, scoreboard_stalls_per_core); - scoreboard_stalls += scoreboard_stalls_per_core; - // alu_stall - uint64_t alu_stalls_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_ALU_ST); - if (num_cores > 1) fprintf(stream, "PERF: core%d: alu unit stalls=%ld\n", core_id, alu_stalls_per_core); - alu_stalls += alu_stalls_per_core; - // lsu_stall - uint64_t lsu_stalls_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_LSU_ST); - if (num_cores > 1) fprintf(stream, "PERF: core%d: lsu unit stalls=%ld\n", core_id, lsu_stalls_per_core); - lsu_stalls += lsu_stalls_per_core; - // fpu_stall - uint64_t fpu_stalls_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_FPU_ST); - if (num_cores > 1) fprintf(stream, "PERF: core%d: fpu unit stalls=%ld\n", core_id, fpu_stalls_per_core); - fpu_stalls += fpu_stalls_per_core; - // sfu_stall - uint64_t sfu_stalls_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SFU_ST); - if (num_cores > 1) fprintf(stream, "PERF: core%d: sfu unit stalls=%ld\n", core_id, sfu_stalls_per_core); - sfu_stalls += sfu_stalls_per_core; + // scheduler idles + { + uint64_t sched_idles_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SCHED_ID); + if (num_cores > 1) { + int idles_percent_per_core = calcAvgPercent(sched_idles_per_core, cycles_per_core); + fprintf(stream, "PERF: core%d: scheduler idle=%ld (%d%%)\n", core_id, sched_idles_per_core, idles_percent_per_core); + } + sched_idles += sched_idles_per_core; + } + // scheduler stalls + { + uint64_t sched_stalls_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SCHED_ST); + if (num_cores > 1) { + int stalls_percent_per_core = calcAvgPercent(sched_stalls_per_core, cycles_per_core); + fprintf(stream, "PERF: core%d: scheduler stalls=%ld (%d%%)\n", core_id, sched_stalls_per_core, stalls_percent_per_core); + } + sched_stalls += sched_stalls_per_core; + } + // ibuffer_stalls + { + uint64_t ibuffer_stalls_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_IBUF_ST); + if (num_cores > 1) { + int ibuffer_percent_per_core = calcAvgPercent(ibuffer_stalls_per_core, cycles_per_core); + fprintf(stream, "PERF: core%d: ibuffer stalls=%ld (%d%%)\n", core_id, ibuffer_stalls_per_core, ibuffer_percent_per_core); + } + ibuffer_stalls += ibuffer_stalls_per_core; + } + // issue_stalls + { + uint64_t scrb_stalls_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SCRB_ST); + uint64_t scrb_alu_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SCRB_ALU); + uint64_t scrb_fpu_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SCRB_FPU); + uint64_t scrb_lsu_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SCRB_LSU); + uint64_t scrb_sfu_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SCRB_SFU); + scrb_alu += scrb_alu_per_core; + scrb_fpu += scrb_fpu_per_core; + scrb_lsu += scrb_lsu_per_core; + scrb_sfu += scrb_sfu_per_core; + if (num_cores > 1) { + uint64_t scrb_total = scrb_alu_per_core + scrb_fpu_per_core + scrb_lsu_per_core + scrb_sfu_per_core; + fprintf(stream, "PERF: core%d: issue stalls=%ld (alu=%d%%, fpu=%d%%, lsu=%d%%, sfu=%d%%)\n", core_id, scrb_stalls_per_core, + calcAvgPercent(scrb_alu_per_core, scrb_total), + calcAvgPercent(scrb_fpu_per_core, scrb_total), + calcAvgPercent(scrb_lsu_per_core, scrb_total), + calcAvgPercent(scrb_sfu_per_core, scrb_total)); + } + scrb_stalls += scrb_stalls_per_core; + } + // sfu_stalls + { + uint64_t scrb_sfu_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SCRB_SFU); + uint64_t scrb_wctl_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SCRB_WCTL); + uint64_t scrb_csrs_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_SCRB_CSRS); + if (num_cores > 1) { + uint64_t sfu_total = scrb_wctl_per_core + scrb_csrs_per_core; + fprintf(stream, "PERF: core%d: sfu stalls=%ld (scrs=%d%%, wctl=%d%%)\n" + , core_id + , scrb_sfu_per_core + , calcAvgPercent(scrb_csrs_per_core, sfu_total) + , calcAvgPercent(scrb_wctl_per_core, sfu_total) + ); + } + scrb_wctl += scrb_wctl_per_core; + scrb_csrs += scrb_csrs_per_core; + } // PERF: memory // ifetches - uint64_t ifetches_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_LOADS); - if (num_cores > 1) fprintf(stream, "PERF: core%d: ifetches=%ld\n", core_id, ifetches_per_core); - ifetches += ifetches_per_core; + { + uint64_t ifetches_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_IFETCHES); + if (num_cores > 1) fprintf(stream, "PERF: core%d: ifetches=%ld\n", core_id, ifetches_per_core); + ifetches += ifetches_per_core; + + uint64_t ifetch_lat_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_IFETCH_LT); + if (num_cores > 1) { + int mem_avg_lat = caclAverage(ifetch_lat_per_core, ifetches_per_core); + fprintf(stream, "PERF: core%d: ifetch latency=%d cycles\n", core_id, mem_avg_lat); + } + ifetch_lat += ifetch_lat_per_core; + } // loads - uint64_t loads_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_LOADS); - if (num_cores > 1) fprintf(stream, "PERF: core%d: loads=%ld\n", core_id, loads_per_core); - loads += loads_per_core; + { + uint64_t loads_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_LOADS); + if (num_cores > 1) fprintf(stream, "PERF: core%d: loads=%ld\n", core_id, loads_per_core); + loads += loads_per_core; + + uint64_t load_lat_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_LOAD_LT); + if (num_cores > 1) { + int mem_avg_lat = caclAverage(load_lat_per_core, loads_per_core); + fprintf(stream, "PERF: core%d: load latency=%d cycles\n", core_id, mem_avg_lat); + } + load_lat += load_lat_per_core; + } // stores - uint64_t stores_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_STORES); - if (num_cores > 1) fprintf(stream, "PERF: core%d: stores=%ld\n", core_id, stores_per_core); - stores += stores_per_core; - // ifetch latency - uint64_t ifetch_lat_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_IFETCH_LAT); - if (num_cores > 1) { - int mem_avg_lat = caclAvgLatency(ifetch_lat_per_core, ifetches_per_core); - fprintf(stream, "PERF: core%d: ifetch latency=%d cycles\n", core_id, mem_avg_lat); + { + uint64_t stores_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_STORES); + if (num_cores > 1) fprintf(stream, "PERF: core%d: stores=%ld\n", core_id, stores_per_core); + stores += stores_per_core; } - ifetch_lat += ifetch_lat_per_core; - // load latency - uint64_t load_lat_per_core = get_csr_64(staging_buf.data(), VX_CSR_MPM_LOAD_LAT); - if (num_cores > 1) { - int mem_avg_lat = caclAvgLatency(load_lat_per_core, loads_per_core); - fprintf(stream, "PERF: core%d: load latency=%d cycles\n", core_id, mem_avg_lat); - } - load_lat += load_lat_per_core; } break; - case VX_DCR_MPM_CLASS_MEM: { + case VX_DCR_MPM_CLASS_MEM: { if (smem_enable) { // PERF: smem uint64_t smem_reads = get_csr_64(staging_buf.data(), VX_CSR_MPM_SMEM_READS); uint64_t smem_writes = get_csr_64(staging_buf.data(), VX_CSR_MPM_SMEM_WRITES); uint64_t smem_bank_stalls = get_csr_64(staging_buf.data(), VX_CSR_MPM_SMEM_BANK_ST); - int smem_bank_utilization = calcUtilization(smem_reads + smem_writes, smem_bank_stalls); + int smem_bank_utilization = calcAvgPercent(smem_reads + smem_writes, smem_reads + smem_writes + smem_bank_stalls); fprintf(stream, "PERF: core%d: smem reads=%ld\n", core_id, smem_reads); fprintf(stream, "PERF: core%d: smem writes=%ld\n", core_id, smem_writes); fprintf(stream, "PERF: core%d: smem bank stalls=%ld (utilization=%d%%)\n", core_id, smem_bank_stalls, smem_bank_utilization); @@ -330,9 +383,12 @@ extern int vx_dump_perf(vx_device_h hdevice, FILE* stream) { // PERF: Icache uint64_t icache_reads = get_csr_64(staging_buf.data(), VX_CSR_MPM_ICACHE_READS); uint64_t icache_read_misses = get_csr_64(staging_buf.data(), VX_CSR_MPM_ICACHE_MISS_R); - int icache_read_hit_ratio = calcRatio(icache_read_misses, icache_reads); + uint64_t icache_mshr_stalls = get_csr_64(staging_buf.data(), VX_CSR_MPM_ICACHE_MSHR_ST); + int icache_read_hit_ratio = calcRatio(icache_read_misses, icache_reads); + int mshr_utilization = calcAvgPercent(icache_read_misses, icache_read_misses + icache_mshr_stalls); fprintf(stream, "PERF: core%d: icache reads=%ld\n", core_id, icache_reads); fprintf(stream, "PERF: core%d: icache read misses=%ld (hit ratio=%d%%)\n", core_id, icache_read_misses, icache_read_hit_ratio); + fprintf(stream, "PERF: core%d: icache mshr stalls=%ld (utilization=%d%%)\n", core_id, icache_mshr_stalls, mshr_utilization); } if (dcache_enable) { @@ -345,13 +401,14 @@ extern int vx_dump_perf(vx_device_h hdevice, FILE* stream) { uint64_t dcache_mshr_stalls = get_csr_64(staging_buf.data(), VX_CSR_MPM_DCACHE_MSHR_ST); int dcache_read_hit_ratio = calcRatio(dcache_read_misses, dcache_reads); int dcache_write_hit_ratio = calcRatio(dcache_write_misses, dcache_writes); - int dcache_bank_utilization = calcUtilization(dcache_reads + dcache_writes, dcache_bank_stalls); + int dcache_bank_utilization = calcAvgPercent(dcache_reads + dcache_writes, dcache_reads + dcache_writes + dcache_bank_stalls); + int mshr_utilization = calcAvgPercent(dcache_read_misses + dcache_write_misses, dcache_read_misses + dcache_write_misses + dcache_mshr_stalls); fprintf(stream, "PERF: core%d: dcache reads=%ld\n", core_id, dcache_reads); fprintf(stream, "PERF: core%d: dcache writes=%ld\n", core_id, dcache_writes); fprintf(stream, "PERF: core%d: dcache read misses=%ld (hit ratio=%d%%)\n", core_id, dcache_read_misses, dcache_read_hit_ratio); fprintf(stream, "PERF: core%d: dcache write misses=%ld (hit ratio=%d%%)\n", core_id, dcache_write_misses, dcache_write_hit_ratio); fprintf(stream, "PERF: core%d: dcache bank stalls=%ld (utilization=%d%%)\n", core_id, dcache_bank_stalls, dcache_bank_utilization); - fprintf(stream, "PERF: core%d: dcache mshr stalls=%ld\n", core_id, dcache_mshr_stalls); + fprintf(stream, "PERF: core%d: dcache mshr stalls=%ld (utilization=%d%%)\n", core_id, dcache_mshr_stalls, mshr_utilization); } if (l2cache_enable) { @@ -378,7 +435,7 @@ extern int vx_dump_perf(vx_device_h hdevice, FILE* stream) { // PERF: memory mem_reads = get_csr_64(staging_buf.data(), VX_CSR_MPM_MEM_READS); mem_writes = get_csr_64(staging_buf.data(), VX_CSR_MPM_MEM_WRITES); - mem_lat = get_csr_64(staging_buf.data(), VX_CSR_MPM_MEM_LAT); + mem_lat = get_csr_64(staging_buf.data(), VX_CSR_MPM_MEM_LT); } } break; default: @@ -386,25 +443,36 @@ extern int vx_dump_perf(vx_device_h hdevice, FILE* stream) { } #endif - uint64_t instrs_per_core = get_csr_64(staging_buf.data(), VX_CSR_MINSTRET); - uint64_t cycles_per_core = get_csr_64(staging_buf.data(), VX_CSR_MCYCLE); float IPC = (float)(double(instrs_per_core) / double(cycles_per_core)); if (num_cores > 1) fprintf(stream, "PERF: core%d: instrs=%ld, cycles=%ld, IPC=%f\n", core_id, instrs_per_core, cycles_per_core, IPC); - instrs += instrs_per_core; - cycles = std::max(cycles_per_core, cycles); + total_instrs += instrs_per_core; + total_cycles += cycles_per_core; + max_cycles = std::max(cycles_per_core, max_cycles); } #ifdef PERF_ENABLE switch (perf_class) { case VX_DCR_MPM_CLASS_CORE: { + int sched_idles_percent = calcAvgPercent(sched_idles, total_cycles); + int sched_stalls_percent = calcAvgPercent(sched_stalls, total_cycles); + int ibuffer_percent = calcAvgPercent(ibuffer_stalls, total_cycles); int ifetch_avg_lat = (int)(double(ifetch_lat) / double(ifetches)); int load_avg_lat = (int)(double(load_lat) / double(loads)); - fprintf(stream, "PERF: ibuffer stalls=%ld\n", ibuffer_stalls); - fprintf(stream, "PERF: scoreboard stalls=%ld\n", scoreboard_stalls); - fprintf(stream, "PERF: alu unit stalls=%ld\n", alu_stalls); - fprintf(stream, "PERF: lsu unit stalls=%ld\n", lsu_stalls); - fprintf(stream, "PERF: fpu unit stalls=%ld\n", fpu_stalls); - fprintf(stream, "PERF: sfu unit stalls=%ld\n", sfu_stalls); + uint64_t scrb_total = scrb_alu + scrb_fpu + scrb_lsu + scrb_sfu; + uint64_t sfu_total = scrb_wctl + scrb_csrs; + fprintf(stream, "PERF: scheduler idle=%ld (%d%%)\n", sched_idles, sched_idles_percent); + fprintf(stream, "PERF: scheduler stalls=%ld (%d%%)\n", sched_stalls, sched_stalls_percent); + fprintf(stream, "PERF: ibuffer stalls=%ld (%d%%)\n", ibuffer_stalls, ibuffer_percent); + fprintf(stream, "PERF: issue stalls=%ld (alu=%d%%, fpu=%d%%, lsu=%d%%, sfu=%d%%)\n", scrb_stalls, + calcAvgPercent(scrb_alu, scrb_total), + calcAvgPercent(scrb_fpu, scrb_total), + calcAvgPercent(scrb_lsu, scrb_total), + calcAvgPercent(scrb_sfu, scrb_total)); + fprintf(stream, "PERF: sfu stalls=%ld (scrs=%d%%, wctl=%d%%)\n" + , scrb_sfu + , calcAvgPercent(scrb_csrs, sfu_total) + , calcAvgPercent(scrb_wctl, sfu_total) + ); fprintf(stream, "PERF: ifetches=%ld\n", ifetches); fprintf(stream, "PERF: loads=%ld\n", loads); fprintf(stream, "PERF: stores=%ld\n", stores); @@ -419,31 +487,32 @@ extern int vx_dump_perf(vx_device_h hdevice, FILE* stream) { l2cache_write_misses /= num_cores; l2cache_bank_stalls /= num_cores; l2cache_mshr_stalls /= num_cores; - int l2cache_read_hit_ratio = calcRatio(l2cache_read_misses, l2cache_reads); - int l2cache_write_hit_ratio = calcRatio(l2cache_write_misses, l2cache_writes); - int l2cache_bank_utilization = calcUtilization(l2cache_reads + l2cache_writes, l2cache_bank_stalls); - + int read_hit_ratio = calcRatio(l2cache_read_misses, l2cache_reads); + int write_hit_ratio = calcRatio(l2cache_write_misses, l2cache_writes); + int bank_utilization = calcAvgPercent(l2cache_reads + l2cache_writes, l2cache_reads + l2cache_writes + l2cache_bank_stalls); + int mshr_utilization = calcAvgPercent(l2cache_read_misses + l2cache_write_misses, l2cache_read_misses + l2cache_write_misses + l2cache_mshr_stalls); fprintf(stream, "PERF: l2cache reads=%ld\n", l2cache_reads); fprintf(stream, "PERF: l2cache writes=%ld\n", l2cache_writes); - fprintf(stream, "PERF: l2cache read misses=%ld (hit ratio=%d%%)\n", l2cache_read_misses, l2cache_read_hit_ratio); - fprintf(stream, "PERF: l2cache write misses=%ld (hit ratio=%d%%)\n", l2cache_write_misses, l2cache_write_hit_ratio); - fprintf(stream, "PERF: l2cache bank stalls=%ld (utilization=%d%%)\n", l2cache_bank_stalls, l2cache_bank_utilization); - fprintf(stream, "PERF: l2cache mshr stalls=%ld\n", l2cache_mshr_stalls); + fprintf(stream, "PERF: l2cache read misses=%ld (hit ratio=%d%%)\n", l2cache_read_misses, read_hit_ratio); + fprintf(stream, "PERF: l2cache write misses=%ld (hit ratio=%d%%)\n", l2cache_write_misses, write_hit_ratio); + fprintf(stream, "PERF: l2cache bank stalls=%ld (utilization=%d%%)\n", l2cache_bank_stalls, bank_utilization); + fprintf(stream, "PERF: l2cache mshr stalls=%ld (utilization=%d%%)\n", l2cache_mshr_stalls, mshr_utilization); } if (l3cache_enable) { - int l3cache_read_hit_ratio = calcRatio(l3cache_read_misses, l3cache_reads); - int l3cache_write_hit_ratio = calcRatio(l3cache_write_misses, l3cache_writes); - int l3cache_bank_utilization = calcUtilization(l3cache_reads + l3cache_writes, l3cache_bank_stalls); + int read_hit_ratio = calcRatio(l3cache_read_misses, l3cache_reads); + int write_hit_ratio = calcRatio(l3cache_write_misses, l3cache_writes); + int bank_utilization = calcAvgPercent(l3cache_reads + l3cache_writes, l3cache_reads + l3cache_writes + l3cache_bank_stalls); + int mshr_utilization = calcAvgPercent(l3cache_read_misses + l3cache_write_misses, l3cache_read_misses + l3cache_write_misses + l3cache_mshr_stalls); fprintf(stream, "PERF: l3cache reads=%ld\n", l3cache_reads); fprintf(stream, "PERF: l3cache writes=%ld\n", l3cache_writes); - fprintf(stream, "PERF: l3cache read misses=%ld (hit ratio=%d%%)\n", l3cache_read_misses, l3cache_read_hit_ratio); - fprintf(stream, "PERF: l3cache write misses=%ld (hit ratio=%d%%)\n", l3cache_write_misses, l3cache_write_hit_ratio); - fprintf(stream, "PERF: l3cache bank stalls=%ld (utilization=%d%%)\n", l3cache_bank_stalls, l3cache_bank_utilization); - fprintf(stream, "PERF: l3cache mshr stalls=%ld\n", l3cache_mshr_stalls); + fprintf(stream, "PERF: l3cache read misses=%ld (hit ratio=%d%%)\n", l3cache_read_misses, read_hit_ratio); + fprintf(stream, "PERF: l3cache write misses=%ld (hit ratio=%d%%)\n", l3cache_write_misses, write_hit_ratio); + fprintf(stream, "PERF: l3cache bank stalls=%ld (utilization=%d%%)\n", l3cache_bank_stalls, bank_utilization); + fprintf(stream, "PERF: l3cache mshr stalls=%ld (utilization=%d%%)\n", l3cache_mshr_stalls, mshr_utilization); } - int mem_avg_lat = caclAvgLatency(mem_lat, mem_reads); + int mem_avg_lat = caclAverage(mem_lat, mem_reads); fprintf(stream, "PERF: memory requests=%ld (reads=%ld, writes=%ld)\n", (mem_reads + mem_writes), mem_reads, mem_writes); fprintf(stream, "PERF: memory latency=%d cycles\n", mem_avg_lat); } break; @@ -452,8 +521,8 @@ extern int vx_dump_perf(vx_device_h hdevice, FILE* stream) { } #endif - float IPC = (float)(double(instrs) / double(cycles)); - fprintf(stream, "PERF: instrs=%ld, cycles=%ld, IPC=%f\n", instrs, cycles, IPC); + float IPC = (float)(double(total_instrs) / double(max_cycles)); + fprintf(stream, "PERF: instrs=%ld, cycles=%ld, IPC=%f\n", total_instrs, max_cycles, IPC); fflush(stream); diff --git a/runtime/opae/.gitignore b/runtime/opae/.gitignore new file mode 100644 index 00000000..541b1f36 --- /dev/null +++ b/runtime/opae/.gitignore @@ -0,0 +1 @@ +/obj_dir/* \ No newline at end of file diff --git a/runtime/opae/Makefile b/runtime/opae/Makefile index 82d0b69c..168d5a11 100644 --- a/runtime/opae/Makefile +++ b/runtime/opae/Makefile @@ -1,25 +1,14 @@ XLEN ?= 32 - TARGET ?= opaesim - -OPAESIM_DIR = ../../sim/opaesim - -RTL_DIR=../../hw/rtl - -SYN_DIR=../../hw/syn/altera/opae - -SCRIPT_DIR=../../hw/scripts +DESTDIR ?= $(CURDIR) +SIM_DIR = $(abspath ../../sim) +HW_DIR = $(abspath ../../hw) +SYN_DIR = $(HW_DIR)/syn/altera/opae CXXFLAGS += -std=c++11 -Wall -Wextra -pedantic -Wfatal-errors -CXXFLAGS += -I. -I../include -I../common/ -I../../hw +CXXFLAGS += -I../include -I../common -I$(HW_DIR) -I$(DESTDIR) CXXFLAGS += -DXLEN_$(XLEN) -ifeq ($(TARGET), opaesim) - CXXFLAGS += -I$(OPAESIM_DIR) -else - CXXFLAGS += -I$(SYN_DIR) -endif - # Position independent code CXXFLAGS += -fPIC @@ -35,9 +24,11 @@ SRCS = vortex.cpp driver.cpp ../common/utils.cpp # set up target types ifeq ($(TARGET), opaesim) - CXXFLAGS += -DOPAESIM - OPAESIM = libopae-c-sim.so + OPAESIM = $(DESTDIR)/libopae-c-sim.so + CXXFLAGS += -DOPAESIM -I$(SIM_DIR)/opaesim + LDFLAGS += -L$(DESTDIR) -lopae-c-sim else + CXXFLAGS += -I$(SYN_DIR) ifeq ($(TARGET), asesim) CXXFLAGS += -DASESIM else @@ -65,14 +56,14 @@ endif PROJECT = libvortex.so -all: $(PROJECT) +all: $(DESTDIR)/$(PROJECT) -libopae-c-sim.so: - DESTDIR=../../runtime/opae $(MAKE) -C $(OPAESIM_DIR) ../../runtime/opae/libopae-c-sim.so +$(DESTDIR)/libopae-c-sim.so: + DESTDIR=$(DESTDIR) $(MAKE) -C $(SIM_DIR)/opaesim $(DESTDIR)/libopae-c-sim.so -$(PROJECT): $(SRCS) $(OPAESIM) - $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $(PROJECT) +$(DESTDIR)/$(PROJECT): $(SRCS) $(OPAESIM) + $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $@ clean: - DESTDIR=../../runtime/opae $(MAKE) -C $(OPAESIM_DIR) clean - rm -rf $(PROJECT) + DESTDIR=$(DESTDIR) $(MAKE) -C $(SIM_DIR)/opaesim clean + rm -rf $(DESTDIR)/$(PROJECT) diff --git a/runtime/rtlsim/.gitignore b/runtime/rtlsim/.gitignore index dc7603c4..541b1f36 100644 --- a/runtime/rtlsim/.gitignore +++ b/runtime/rtlsim/.gitignore @@ -1,2 +1 @@ -obj_dir -*.so +/obj_dir/* \ No newline at end of file diff --git a/runtime/rtlsim/Makefile b/runtime/rtlsim/Makefile index 57f58df4..5f85bdc3 100644 --- a/runtime/rtlsim/Makefile +++ b/runtime/rtlsim/Makefile @@ -1,9 +1,10 @@ XLEN ?= 32 - -RTLSIM_DIR = ../../sim/rtlsim +DESTDIR ?= $(CURDIR) +SIM_DIR = $(abspath ../../sim) +HW_DIR = $(abspath ../../hw) CXXFLAGS += -std=c++11 -Wall -Wextra -pedantic -Wfatal-errors -CXXFLAGS += -I../include -I../common -I../../hw -I$(RTLSIM_DIR) -I$(RTLSIM_DIR)/../common +CXXFLAGS += -I../include -I../common -I$(HW_DIR) -I$(SIM_DIR)/rtlsim -I$(SIM_DIR)/common CXXFLAGS += -DXLEN_$(XLEN) # Position independent code @@ -16,7 +17,7 @@ CXXFLAGS += $(CONFIGS) CXXFLAGS += -DDUMP_PERF_STATS LDFLAGS += -shared -pthread -LDFLAGS += -L. -lrtlsim +LDFLAGS += -L$(DESTDIR) -lrtlsim SRCS = vortex.cpp ../common/utils.cpp @@ -34,12 +35,12 @@ endif PROJECT = libvortex.so -all: $(PROJECT) +all: $(DESTDIR)/$(PROJECT) -$(PROJECT): $(SRCS) - DESTDIR=../../runtime/rtlsim $(MAKE) -C $(RTLSIM_DIR) ../../runtime/rtlsim/librtlsim.so - $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $(PROJECT) +$(DESTDIR)/$(PROJECT): $(SRCS) + DESTDIR=$(DESTDIR) $(MAKE) -C $(SIM_DIR)/rtlsim $(DESTDIR)/librtlsim.so + $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $@ clean: - DESTDIR=../../runtime/rtlsim $(MAKE) -C $(RTLSIM_DIR) clean - rm -rf $(PROJECT) *.o \ No newline at end of file + DESTDIR=$(DESTDIR) $(MAKE) -C $(SIM_DIR)/rtlsim clean + rm -rf $(DESTDIR)/$(PROJECT) *.o \ No newline at end of file diff --git a/runtime/simx/Makefile b/runtime/simx/Makefile index c9f157fd..7cfd6c38 100644 --- a/runtime/simx/Makefile +++ b/runtime/simx/Makefile @@ -1,16 +1,17 @@ XLEN ?= 32 - -SIMX_DIR = ../../sim/simx +DESTDIR ?= $(CURDIR) +SIM_DIR = $(abspath ../../sim) +HW_DIR = $(abspath ../../hw) CXXFLAGS += -std=c++11 -Wall -Wextra -Wfatal-errors CXXFLAGS += -fPIC -Wno-maybe-uninitialized -CXXFLAGS += -I../include -I../common -I../../hw -I$(SIMX_DIR) -I$(SIMX_DIR)/../common +CXXFLAGS += -I../include -I../common -I$(HW_DIR) -I$(SIM_DIR)/simx -I$(SIM_DIR)/common CXXFLAGS += $(CONFIGS) CXXFLAGS += -DDUMP_PERF_STATS CXXFLAGS += -DXLEN_$(XLEN) LDFLAGS += -shared -pthread -LDFLAGS += -L. -lsimx +LDFLAGS += -L$(DESTDIR) -lsimx SRCS = vortex.cpp ../common/utils.cpp @@ -23,12 +24,12 @@ endif PROJECT = libvortex.so -all: $(PROJECT) +all: $(DESTDIR)/$(PROJECT) -$(PROJECT): $(SRCS) - DESTDIR=../../runtime/simx $(MAKE) -C $(SIMX_DIR) ../../runtime/simx/libsimx.so +$(DESTDIR)/$(PROJECT): $(SRCS) + DESTDIR=$(DESTDIR) $(MAKE) -C $(SIM_DIR)/simx $(DESTDIR)/libsimx.so $(CXX) $(CXXFLAGS) $^ $(LDFLAGS) -o $@ clean: - DESTDIR=../../runtime/simx $(MAKE) -C $(SIMX_DIR) clean - rm -rf libsimx.so $(PROJECT) *.o \ No newline at end of file + DESTDIR=$(DESTDIR) $(MAKE) -C $(SIM_DIR)/simx clean + rm -rf $(DESTDIR)/$(PROJECT) *.o \ No newline at end of file diff --git a/runtime/simx/vortex.cpp b/runtime/simx/vortex.cpp index 3b4cb171..b7b9cdcb 100644 --- a/runtime/simx/vortex.cpp +++ b/runtime/simx/vortex.cpp @@ -87,7 +87,7 @@ private: class vx_device { public: vx_device() - : arch_(NUM_THREADS, NUM_WARPS, NUM_CORES, NUM_CLUSTERS) + : arch_(NUM_THREADS, NUM_WARPS, NUM_CORES) , ram_(RAM_PAGE_SIZE) , processor_(arch_) , global_mem_( diff --git a/runtime/stub/Makefile b/runtime/stub/Makefile index 28441a3d..9c1c40bd 100644 --- a/runtime/stub/Makefile +++ b/runtime/stub/Makefile @@ -1,6 +1,11 @@ +XLEN ?= 32 +DESTDIR ?= $(CURDIR) +SIM_DIR = $(abspath ../../sim) +HW_DIR = $(abspath ../../hw) + CXXFLAGS += -std=c++11 -O2 -Wall -Wextra -pedantic -Wfatal-errors -CXXFLAGS += -I../include -I../../runtime -I../../hw -I../../sim/common +CXXFLAGS += -I../include -I../common -I$(HW_DIR) -I$(SIM_DIR)/common CXXFLAGS += -fPIC diff --git a/sim/opaesim/Makefile b/sim/opaesim/Makefile index 3b906bc5..2b014711 100644 --- a/sim/opaesim/Makefile +++ b/sim/opaesim/Makefile @@ -1,20 +1,22 @@ XLEN ?= 32 -DESTDIR ?= . -RTL_DIR = ../../hw/rtl -DPI_DIR = ../../hw/dpi +DESTDIR ?= $(CURDIR) +HW_DIR = $(abspath ../../hw) +COMMON_DIR = $(abspath ../common) +THIRD_PARTY_DIR = $(abspath ../../third_party) +RTL_DIR = $(HW_DIR)/rtl +DPI_DIR = $(HW_DIR)/dpi AFU_DIR = $(RTL_DIR)/afu/opae -SCRIPT_DIR = ../../hw/scripts -THIRD_PARTY_DIR = ../../third_party +SCRIPT_DIR = $(HW_DIR)/scripts CXXFLAGS += -std=c++11 -Wall -Wextra -Wfatal-errors -Wno-array-bounds CXXFLAGS += -fPIC -Wno-maybe-uninitialized -CXXFLAGS += -I.. -I../../../hw -I../../common -I$(abspath $(DESTDIR)) -CXXFLAGS += -I../$(THIRD_PARTY_DIR)/softfloat/source/include -CXXFLAGS += -I../$(THIRD_PARTY_DIR) +CXXFLAGS += -I$(CURDIR) -I$(HW_DIR) -I$(COMMON_DIR) -I$(DESTDIR) +CXXFLAGS += -I/$(THIRD_PARTY_DIR)/softfloat/source/include +CXXFLAGS += -I/$(THIRD_PARTY_DIR) CXXFLAGS += -DXLEN_$(XLEN) -LDFLAGS += -shared ../$(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -L../$(THIRD_PARTY_DIR)/ramulator -lramulator -pthread +LDFLAGS += -shared $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a +LDFLAGS += -L$(THIRD_PARTY_DIR)/ramulator -lramulator -pthread # control RTL debug tracing states DBG_TRACE_FLAGS += -DDBG_TRACE_CORE_PIPELINE @@ -53,9 +55,9 @@ endif DBG_FLAGS += -DDEBUG_LEVEL=$(DEBUG) -DVCD_OUTPUT $(DBG_TRACE_FLAGS) -SRCS = ../common/util.cpp ../common/mem.cpp ../common/rvfloats.cpp +SRCS = $(COMMON_DIR)/util.cpp $(COMMON_DIR)/mem.cpp $(COMMON_DIR)/rvfloats.cpp SRCS += $(DPI_DIR)/util_dpi.cpp $(DPI_DIR)/float_dpi.cpp -SRCS += fpga.cpp opae_sim.cpp +SRCS += $(CURDIR)/fpga.cpp $(CURDIR)/opae_sim.cpp RTL_PKGS = $(AFU_DIR)/local_mem_cfg_pkg.sv $(AFU_DIR)/ccip/ccip_if_pkg.sv RTL_PKGS += $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/fpu/VX_fpu_pkg.sv @@ -73,7 +75,7 @@ TOP = vortex_afu_shim VL_FLAGS += --language 1800-2009 --assert -Wall -Wpedantic VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique -VL_FLAGS += -DSIMULATION +VL_FLAGS += -DSIMULATION -DSV_DPI VL_FLAGS += -DXLEN_$(XLEN) VL_FLAGS += $(CONFIGS) VL_FLAGS += verilator.vlt @@ -119,16 +121,16 @@ PROJECT = libopae-c-sim.so all: $(DESTDIR)/$(PROJECT) $(DESTDIR)/vortex.xml: - verilator --xml-only -O0 $(VL_FLAGS) $(TOP) --xml-output $(DESTDIR)/vortex.xml + verilator --xml-only -O0 $(VL_FLAGS) $(TOP) --xml-output $@ $(DESTDIR)/scope.json: $(DESTDIR)/vortex.xml - $(SCRIPT_DIR)/scope.py $(DESTDIR)/vortex.xml -o $(DESTDIR)/scope.json + $(SCRIPT_DIR)/scope.py $^ -o $@ $(DESTDIR)/vortex_afu.h : $(AFU_DIR)/vortex_afu.vh - $(SCRIPT_DIR)/gen_config.py -i $(AFU_DIR)/vortex_afu.vh -o $(DESTDIR)/vortex_afu.h + $(SCRIPT_DIR)/gen_config.py -i $^ -o $@ $(DESTDIR)/$(PROJECT): $(SRCS) $(DESTDIR)/vortex_afu.h $(SCOPE_JSON) - verilator --build --exe -O3 $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' -o ../$(DESTDIR)/$(PROJECT) + verilator --build --exe -O3 $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $(DESTDIR)/obj_dir -o $@ clean: - rm -rf obj_dir $(DESTDIR)/vortex.xml $(DESTDIR)/scope.json $(DESTDIR)/vortex_afu.h $(DESTDIR)/$(PROJECT) + rm -rf $(DESTDIR)/obj_dir $(DESTDIR)/vortex.xml $(DESTDIR)/scope.json $(DESTDIR)/vortex_afu.h $(DESTDIR)/$(PROJECT) diff --git a/sim/rtlsim/.gitignore b/sim/rtlsim/.gitignore index d24f77ea..541b1f36 100644 --- a/sim/rtlsim/.gitignore +++ b/sim/rtlsim/.gitignore @@ -1,2 +1 @@ -VX_config.h /obj_dir/* \ No newline at end of file diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index 1d43ea4f..d3a49fbb 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -1,18 +1,20 @@ XLEN ?= 32 -DESTDIR ?= . -RTL_DIR = ../../hw/rtl -DPI_DIR = ../../hw/dpi -THIRD_PARTY_DIR = ../../third_party +DESTDIR ?= $(CURDIR) +HW_DIR = $(abspath ../../hw) +COMMON_DIR = $(abspath ../common) +THIRD_PARTY_DIR = $(abspath ../../third_party) +RTL_DIR = $(HW_DIR)/rtl +DPI_DIR = $(HW_DIR)/dpi CXXFLAGS += -std=c++11 -Wall -Wextra -Wfatal-errors -Wno-array-bounds CXXFLAGS += -fPIC -Wno-maybe-uninitialized -CXXFLAGS += -I../../../hw -I../../common -CXXFLAGS += -I../$(THIRD_PARTY_DIR)/softfloat/source/include -CXXFLAGS += -I../$(THIRD_PARTY_DIR) +CXXFLAGS += -I$(HW_DIR) -I$(COMMON_DIR) +CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include +CXXFLAGS += -I$(THIRD_PARTY_DIR) CXXFLAGS += -DXLEN_$(XLEN) -LDFLAGS += ../$(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -L../$(THIRD_PARTY_DIR)/ramulator -lramulator +LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a +LDFLAGS += -L$(THIRD_PARTY_DIR)/ramulator -lramulator # control RTL debug tracing states DBG_TRACE_FLAGS += -DDBG_TRACE_CORE_PIPELINE @@ -38,9 +40,9 @@ ifneq (,$(findstring FPU_FPNEW,$(CONFIGS))) endif RTL_INCLUDE = -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/cache $(FPU_INCLUDE) -SRCS = ../common/util.cpp ../common/mem.cpp ../common/rvfloats.cpp +SRCS = $(COMMON_DIR)/util.cpp $(COMMON_DIR)/mem.cpp $(COMMON_DIR)/rvfloats.cpp SRCS += $(DPI_DIR)/util_dpi.cpp $(DPI_DIR)/float_dpi.cpp -SRCS += processor.cpp +SRCS += $(CURDIR)/processor.cpp ifdef AXI_BUS TOP = Vortex_axi @@ -54,7 +56,7 @@ VL_FLAGS += --language 1800-2009 --assert -Wall -Wpedantic VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt -VL_FLAGS += -DSIMULATION +VL_FLAGS += -DSIMULATION -DSV_DPI VL_FLAGS += -DXLEN_$(XLEN) VL_FLAGS += $(CONFIGS) VL_FLAGS += $(RTL_INCLUDE) @@ -87,11 +89,11 @@ PROJECT = rtlsim all: $(DESTDIR)/$(PROJECT) -$(DESTDIR)/$(PROJECT): $(SRCS) main.cpp - verilator --build $(VL_FLAGS) $^ -CFLAGS '$(CXXFLAGS) -DSTARTUP_ADDR=0x80000000' -LDFLAGS '$(LDFLAGS)' -o ../$@ +$(DESTDIR)/$(PROJECT): $(SRCS) $(CURDIR)/main.cpp + verilator --build $(VL_FLAGS) $^ -CFLAGS '$(CXXFLAGS) -DSTARTUP_ADDR=0x80000000' -LDFLAGS '$(LDFLAGS)' --Mdir $(DESTDIR)/obj_dir -o $@ $(DESTDIR)/lib$(PROJECT).so: $(SRCS) - verilator --build $(VL_FLAGS) $^ -CFLAGS '$(CXXFLAGS)' -LDFLAGS '-shared $(LDFLAGS)' -o ../$@ + verilator --build $(VL_FLAGS) $^ -CFLAGS '$(CXXFLAGS)' -LDFLAGS '-shared $(LDFLAGS)' --Mdir $(DESTDIR)/obj_dir -o $@ clean: - rm -rf obj_dir $(DESTDIR)/$(PROJECT) $(DESTDIR)/lib$(PROJECT).so + rm -rf $(DESTDIR)/obj_dir $(DESTDIR)/$(PROJECT) $(DESTDIR)/lib$(PROJECT).so diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 42823205..36f0b63e 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -1,11 +1,12 @@ XLEN ?= 32 -DESTDIR ?= . -RTL_DIR = ../hw/rtl -THIRD_PARTY_DIR = ../../third_party +DESTDIR ?= $(CURDIR) +HW_DIR = $(abspath ../../hw) +COMMON_DIR = $(abspath ../common) +THIRD_PARTY_DIR = $(abspath ../../third_party) CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors CXXFLAGS += -fPIC -Wno-maybe-uninitialized -CXXFLAGS += -I. -I../common -I../../hw +CXXFLAGS += -I$(CURDIR) -I$(COMMON_DIR) -I$(HW_DIR) CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include CXXFLAGS += -I$(THIRD_PARTY_DIR) CXXFLAGS += -DXLEN_$(XLEN) @@ -14,8 +15,8 @@ CXXFLAGS += $(CONFIGS) LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a LDFLAGS += -L$(THIRD_PARTY_DIR)/ramulator -lramulator -SRCS = ../common/util.cpp ../common/mem.cpp ../common/rvfloats.cpp -SRCS += processor.cpp cluster.cpp core.cpp warp.cpp decode.cpp execute.cpp exe_unit.cpp cache_sim.cpp mem_sim.cpp shared_mem.cpp dcrs.cpp +SRCS = $(COMMON_DIR)/util.cpp $(COMMON_DIR)/mem.cpp $(COMMON_DIR)/rvfloats.cpp +SRCS += processor.cpp cluster.cpp socket.cpp core.cpp warp.cpp decode.cpp execute.cpp exe_unit.cpp cache_sim.cpp mem_sim.cpp shared_mem.cpp dcrs.cpp # Debugigng ifdef DEBUG diff --git a/sim/simx/arch.h b/sim/simx/arch.h index ab6ac4a3..099fbedd 100644 --- a/sim/simx/arch.h +++ b/sim/simx/arch.h @@ -28,6 +28,7 @@ private: uint16_t num_warps_; uint16_t num_cores_; uint16_t num_clusters_; + uint16_t socket_size_; uint16_t vsize_; uint16_t num_regs_; uint16_t num_csrs_; @@ -35,11 +36,12 @@ private: uint16_t ipdom_size_; public: - Arch(uint16_t num_threads, uint16_t num_warps, uint16_t num_cores, uint16_t num_clusters) + Arch(uint16_t num_threads, uint16_t num_warps, uint16_t num_cores) : num_threads_(num_threads) , num_warps_(num_warps) , num_cores_(num_cores) - , num_clusters_(num_clusters) + , num_clusters_(NUM_CLUSTERS) + , socket_size_(SOCKET_SIZE) , vsize_(16) , num_regs_(32) , num_csrs_(4096) @@ -82,6 +84,10 @@ public: uint16_t num_clusters() const { return num_clusters_; } + + uint16_t socket_size() const { + return socket_size_; + } }; } \ No newline at end of file diff --git a/sim/simx/cache_cluster.h b/sim/simx/cache_cluster.h index be3146d3..aef28d1a 100644 --- a/sim/simx/cache_cluster.h +++ b/sim/simx/cache_cluster.h @@ -45,20 +45,20 @@ public: char sname[100]; - std::vector::Ptr> unit_arbs(num_units); + std::vector unit_arbs(num_units); for (uint32_t u = 0; u < num_units; ++u) { snprintf(sname, 100, "%s-unit-arb-%d", name, u); - unit_arbs.at(u) = Switch::Create(sname, ArbiterType::RoundRobin, num_requests, config.num_inputs); + unit_arbs.at(u) = MemSwitch::Create(sname, ArbiterType::RoundRobin, num_requests, config.num_inputs); for (uint32_t i = 0; i < num_requests; ++i) { this->CoreReqPorts.at(u).at(i).bind(&unit_arbs.at(u)->ReqIn.at(i)); unit_arbs.at(u)->RspIn.at(i).bind(&this->CoreRspPorts.at(u).at(i)); } } - std::vector::Ptr> mem_arbs(config.num_inputs); + std::vector mem_arbs(config.num_inputs); for (uint32_t i = 0; i < config.num_inputs; ++i) { snprintf(sname, 100, "%s-mem-arb-%d", name, i); - mem_arbs.at(i) = Switch::Create(sname, ArbiterType::RoundRobin, num_units, num_caches); + mem_arbs.at(i) = MemSwitch::Create(sname, ArbiterType::RoundRobin, num_units, num_caches); for (uint32_t u = 0; u < num_units; ++u) { unit_arbs.at(u)->ReqOut.at(i).bind(&mem_arbs.at(i)->ReqIn.at(u)); mem_arbs.at(i)->RspIn.at(u).bind(&unit_arbs.at(u)->RspOut.at(i)); @@ -66,7 +66,7 @@ public: } snprintf(sname, 100, "%s-cache-arb", name); - auto cache_arb = Switch::Create(sname, ArbiterType::RoundRobin, num_caches, 1); + auto cache_arb = MemSwitch::Create(sname, ArbiterType::RoundRobin, num_caches, 1); for (uint32_t i = 0; i < num_caches; ++i) { snprintf(sname, 100, "%s-cache%d", name, i); diff --git a/sim/simx/cache_sim.cpp b/sim/simx/cache_sim.cpp index 5a6906a9..2c9410f4 100644 --- a/sim/simx/cache_sim.cpp +++ b/sim/simx/cache_sim.cpp @@ -41,19 +41,16 @@ struct params_t { uint32_t tag_select_addr_end; params_t(const CacheSim::Config& config) { - int32_t bank_bits = log2ceil(config.num_banks); - int32_t offset_bits = config.B - config.W; - int32_t log2_bank_size = config.C - bank_bits; - int32_t index_bits = log2_bank_size - (config.B + config.A); - assert(log2_bank_size > 0); + int32_t offset_bits = config.L - config.W; + int32_t index_bits = config.C - (config.L + config.A + config.B); assert(offset_bits >= 0); assert(index_bits >= 0); this->log2_num_inputs = log2ceil(config.num_inputs); - this->words_per_line = 1 << offset_bits; + this->sets_per_bank = 1 << index_bits; this->lines_per_set = 1 << config.A; - this->sets_per_bank = 1 << index_bits; + this->words_per_line = 1 << offset_bits; assert(config.ports_per_bank <= this->words_per_line); @@ -63,7 +60,7 @@ struct params_t { // Bank select this->bank_select_addr_start = (1+this->word_select_addr_end); - this->bank_select_addr_end = (this->bank_select_addr_start+bank_bits-1); + this->bank_select_addr_end = (this->bank_select_addr_start+config.B-1); // Set select this->set_select_addr_start = (1+this->bank_select_addr_end); @@ -74,23 +71,23 @@ struct params_t { this->tag_select_addr_end = (config.addr_width-1); } - uint32_t addr_bank_id(uint64_t word_addr) const { + uint32_t addr_bank_id(uint64_t addr) const { if (bank_select_addr_end >= bank_select_addr_start) - return (uint32_t)bit_getw(word_addr, bank_select_addr_start, bank_select_addr_end); + return (uint32_t)bit_getw(addr, bank_select_addr_start, bank_select_addr_end); else return 0; } - uint32_t addr_set_id(uint64_t word_addr) const { + uint32_t addr_set_id(uint64_t addr) const { if (set_select_addr_end >= set_select_addr_start) - return (uint32_t)bit_getw(word_addr, set_select_addr_start, set_select_addr_end); + return (uint32_t)bit_getw(addr, set_select_addr_start, set_select_addr_end); else return 0; } - uint64_t addr_tag(uint64_t word_addr) const { + uint64_t addr_tag(uint64_t addr) const { if (tag_select_addr_end >= tag_select_addr_start) - return bit_getw(word_addr, tag_select_addr_start, tag_select_addr_end); + return bit_getw(addr, tag_select_addr_start, tag_select_addr_end); else return 0; } @@ -288,8 +285,8 @@ private: Config config_; params_t params_; std::vector banks_; - Switch::Ptr bank_switch_; - Switch::Ptr bypass_switch_; + MemSwitch::Ptr bank_switch_; + MemSwitch::Ptr bypass_switch_; std::vector> mem_req_ports_; std::vector> mem_rsp_ports_; std::vector pipeline_reqs_; @@ -304,16 +301,16 @@ public: : simobject_(simobject) , config_(config) , params_(config) - , banks_(config.num_banks, {config, params_}) - , mem_req_ports_(config.num_banks, simobject) - , mem_rsp_ports_(config.num_banks, simobject) - , pipeline_reqs_(config.num_banks, config.ports_per_bank) + , banks_((1 << config.B), {config, params_}) + , mem_req_ports_((1 << config.B), simobject) + , mem_rsp_ports_((1 << config.B), simobject) + , pipeline_reqs_((1 << config.B), config.ports_per_bank) { char sname[100]; snprintf(sname, 100, "%s-bypass-arb", simobject->name().c_str()); if (config_.bypass) { - bypass_switch_ = Switch::Create(sname, ArbiterType::RoundRobin, config_.num_inputs); + bypass_switch_ = MemSwitch::Create(sname, ArbiterType::RoundRobin, config_.num_inputs); for (uint32_t i = 0; i < config_.num_inputs; ++i) { simobject->CoreReqPorts.at(i).bind(&bypass_switch_->ReqIn.at(i)); bypass_switch_->RspIn.at(i).bind(&simobject->CoreRspPorts.at(i)); @@ -323,14 +320,14 @@ public: return; } - bypass_switch_ = Switch::Create(sname, ArbiterType::Priority, 2); + bypass_switch_ = MemSwitch::Create(sname, ArbiterType::Priority, 2); bypass_switch_->ReqOut.at(0).bind(&simobject->MemReqPort); simobject->MemRspPort.bind(&bypass_switch_->RspOut.at(0)); - if (config.num_banks > 1) { + if (config.B != 0) { snprintf(sname, 100, "%s-bank-arb", simobject->name().c_str()); - bank_switch_ = Switch::Create(sname, ArbiterType::RoundRobin, config.num_banks); - for (uint32_t i = 0, n = config.num_banks; i < n; ++i) { + bank_switch_ = MemSwitch::Create(sname, ArbiterType::RoundRobin, (1 << config.B)); + for (uint32_t i = 0, n = (1 << config.B); i < n; ++i) { mem_req_ports_.at(i).bind(&bank_switch_->ReqIn.at(i)); bank_switch_->RspIn.at(i).bind(&mem_rsp_ports_.at(i)); } @@ -383,20 +380,22 @@ public: pipeline_req.clear(); } - // schedule MSHR replay - for (uint32_t bank_id = 0, n = config_.num_banks; bank_id < n; ++bank_id) { + // first: schedule MSHR replay (flush MSHR queue) + for (uint32_t bank_id = 0, n = (1 << config_.B); bank_id < n; ++bank_id) { auto& bank = banks_.at(bank_id); auto& pipeline_req = pipeline_reqs_.at(bank_id); bank.mshr.pop(&pipeline_req); } - // schedule memory fill - for (uint32_t bank_id = 0, n = config_.num_banks; bank_id < n; ++bank_id) { + // second: schedule memory fill (flush memory queue) + for (uint32_t bank_id = 0, n = (1 << config_.B); bank_id < n; ++bank_id) { auto& mem_rsp_port = mem_rsp_ports_.at(bank_id); if (mem_rsp_port.empty()) continue; auto& pipeline_req = pipeline_reqs_.at(bank_id); + + // skip if bank already busy if (pipeline_req.type != bank_req_t::None) continue; @@ -407,7 +406,7 @@ public: mem_rsp_port.pop(); } - // schedule core requests + // last: schedule core requests (flush core queue) for (uint32_t req_id = 0, n = config_.num_inputs; req_id < n; ++req_id) { auto& core_req_port = simobject_->CoreReqPorts.at(req_id); if (core_req_port.empty()) @@ -425,18 +424,21 @@ public: } auto bank_id = params_.addr_bank_id(core_req.addr); - auto set_id = params_.addr_set_id(core_req.addr); - auto tag = params_.addr_tag(core_req.addr); - auto port_id = req_id % config_.ports_per_bank; - auto& bank = banks_.at(bank_id); auto& pipeline_req = pipeline_reqs_.at(bank_id); + // skip if bank already busy + if (pipeline_req.type != bank_req_t::None) + continue; + + auto set_id = params_.addr_set_id(core_req.addr); + auto tag = params_.addr_tag(core_req.addr); + auto port_id = req_id % config_.ports_per_bank; + // check MSHR capacity if ((!core_req.write || !config_.write_through) && bank.mshr.full()) { ++perf_stats_.mshr_stalls; - ++perf_stats_.bank_stalls; continue; } @@ -452,7 +454,7 @@ public: } // extend request ports pipeline_req.ports.at(port_id) = bank_req_port_t{req_id, core_req.tag, true}; - } else if (pipeline_req.type == bank_req_t::None) { + } else { // schedule new request bank_req_t bank_req(config_.ports_per_bank); bank_req.ports.at(port_id) = bank_req_port_t{req_id, core_req.tag, true}; @@ -463,10 +465,6 @@ public: bank_req.type = bank_req_t::Core; bank_req.write = core_req.write; pipeline_req = bank_req; - } else { - // bank in use - ++perf_stats_.bank_stalls; - continue; } if (core_req.write) @@ -516,7 +514,7 @@ private: } void processBankRequests() { - for (uint32_t bank_id = 0, n = config_.num_banks; bank_id < n; ++bank_id) { + for (uint32_t bank_id = 0, n = (1 << config_.B); bank_id < n; ++bank_id) { auto& bank = banks_.at(bank_id); auto pipeline_req = pipeline_reqs_.at(bank_id); @@ -545,11 +543,10 @@ private: } } } break; - case bank_req_t::Core: { - bool hit = false; - bool found_free_line = false; - uint32_t hit_line_id = 0; - uint32_t repl_line_id = 0; + case bank_req_t::Core: { + int32_t hit_line_id = -1; + int32_t free_line_id = -1; + int32_t repl_line_id = 0; uint32_t max_cnt = 0; auto& set = bank.sets.at(pipeline_req.set_id); @@ -557,38 +554,34 @@ private: // tag lookup for (uint32_t i = 0, n = set.lines.size(); i < n; ++i) { auto& line = set.lines.at(i); + if (max_cnt < line.lru_ctr) { + max_cnt = line.lru_ctr; + repl_line_id = i; + } if (line.valid) { - if (line.tag == pipeline_req.tag) { - line.lru_ctr = 0; + if (line.tag == pipeline_req.tag) { hit_line_id = i; - hit = true; + line.lru_ctr = 0; } else { ++line.lru_ctr; } - if (max_cnt < line.lru_ctr) { - max_cnt = line.lru_ctr; - repl_line_id = i; - } } else { - found_free_line = true; - repl_line_id = i; + free_line_id = i; } } - if (hit) { - // - // Hit handling - // + if (hit_line_id != -1) { + // Hit handling if (pipeline_req.write) { - // handle write hit + // handle write has_hit auto& hit_line = set.lines.at(hit_line_id); if (config_.write_through) { // forward write request to memory MemReq mem_req; - mem_req.addr = params_.mem_addr(bank_id, pipeline_req.set_id, hit_line.tag); + mem_req.addr = params_.mem_addr(bank_id, pipeline_req.set_id, pipeline_req.tag); mem_req.write = true; - mem_req.cid = pipeline_req.cid; - mem_req.uuid = pipeline_req.uuid; + mem_req.cid = pipeline_req.cid; + mem_req.uuid = pipeline_req.uuid; mem_req_ports_.at(bank_id).send(mem_req, 1); DT(3, simobject_->name() << "-dram-" << mem_req); } else { @@ -606,23 +599,21 @@ private: DT(3, simobject_->name() << "-core-" << core_rsp); } } - } else { - // - // Miss handling - // + } else { + // Miss handling if (pipeline_req.write) ++perf_stats_.write_misses; else ++perf_stats_.read_misses; - if (!found_free_line && !config_.write_through) { + if (free_line_id == -1 && !config_.write_through) { // write back dirty line auto& repl_line = set.lines.at(repl_line_id); if (repl_line.dirty) { MemReq mem_req; mem_req.addr = params_.mem_addr(bank_id, pipeline_req.set_id, repl_line.tag); mem_req.write = true; - mem_req.cid = pipeline_req.cid; + mem_req.cid = pipeline_req.cid; mem_req_ports_.at(bank_id).send(mem_req, 1); DT(3, simobject_->name() << "-dram-" << mem_req); ++perf_stats_.evictions; @@ -635,8 +626,8 @@ private: MemReq mem_req; mem_req.addr = params_.mem_addr(bank_id, pipeline_req.set_id, pipeline_req.tag); mem_req.write = true; - mem_req.cid = pipeline_req.cid; - mem_req.uuid = pipeline_req.uuid; + mem_req.cid = pipeline_req.cid; + mem_req.uuid = pipeline_req.uuid; mem_req_ports_.at(bank_id).send(mem_req, 1); DT(3, simobject_->name() << "-dram-" << mem_req); } @@ -655,7 +646,7 @@ private: auto mshr_pending = bank.mshr.lookup(pipeline_req); // allocate MSHR - auto mshr_id = bank.mshr.allocate(pipeline_req, repl_line_id); + auto mshr_id = bank.mshr.allocate(pipeline_req, (free_line_id != -1) ? free_line_id : repl_line_id); // send fill request if (!mshr_pending) { @@ -663,8 +654,8 @@ private: mem_req.addr = params_.mem_addr(bank_id, pipeline_req.set_id, pipeline_req.tag); mem_req.write = false; mem_req.tag = mshr_id; - mem_req.cid = pipeline_req.cid; - mem_req.uuid = pipeline_req.uuid; + mem_req.cid = pipeline_req.cid; + mem_req.uuid = pipeline_req.uuid; mem_req_ports_.at(bank_id).send(mem_req, 1); DT(3, simobject_->name() << "-dram-" << mem_req); ++pending_fill_reqs_; diff --git a/sim/simx/cache_sim.h b/sim/simx/cache_sim.h index 498fb73f..2faea91d 100644 --- a/sim/simx/cache_sim.h +++ b/sim/simx/cache_sim.h @@ -23,16 +23,15 @@ public: struct Config { bool bypass; // cache bypass uint8_t C; // log2 cache size - uint8_t B; // log2 block size + uint8_t L; // log2 line size uint8_t W; // log2 word size uint8_t A; // log2 associativity - uint8_t addr_width; // word address bits - uint8_t num_banks; // number of banks + uint8_t B; // log2 number of banks + uint8_t addr_width; // word address bits uint8_t ports_per_bank; // number of ports per bank uint8_t num_inputs; // number of inputs bool write_through; // is write-through bool write_reponse; // enable write response - uint16_t victim_size; // victim cache size uint16_t mshr_size; // MSHR buffer size uint8_t latency; // pipeline latency }; diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index a129ddaa..3ac80cb6 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -18,34 +18,60 @@ using namespace vortex; Cluster::Cluster(const SimContext& ctx, uint32_t cluster_id, ProcessorImpl* processor, - const Arch &arch, const - DCRS &dcrs) + const Arch &arch, + const DCRS &dcrs) : SimObject(ctx, "cluster") , mem_req_port(this) , mem_rsp_port(this) , cluster_id_(cluster_id) - , cores_(arch.num_cores()) - , barriers_(arch.num_barriers(), 0) - , sharedmems_(arch.num_cores()) , processor_(processor) + , sockets_(NUM_SOCKETS) + , barriers_(arch.num_barriers(), 0) + , cores_per_socket_(arch.socket_size()) { - auto num_cores = arch.num_cores(); - char sname[100]; + + uint32_t sockets_per_cluster = sockets_.size(); + + // create sockets + + snprintf(sname, 100, "cluster%d-icache-arb", cluster_id); + auto icache_switch = MemSwitch::Create(sname, ArbiterType::RoundRobin, sockets_per_cluster); + + snprintf(sname, 100, "cluster%d-dcache-arb", cluster_id); + auto dcache_switch = MemSwitch::Create(sname, ArbiterType::RoundRobin, sockets_per_cluster); + + for (uint32_t i = 0; i < sockets_per_cluster; ++i) { + uint32_t socket_id = cluster_id * sockets_per_cluster + i; + auto socket = Socket::Create(socket_id, + this, + arch, + dcrs); + + socket->icache_mem_req_port.bind(&icache_switch->ReqIn.at(i)); + icache_switch->RspIn.at(i).bind(&socket->icache_mem_rsp_port); + + socket->dcache_mem_req_port.bind(&dcache_switch->ReqIn.at(i)); + dcache_switch->RspIn.at(i).bind(&socket->dcache_mem_rsp_port); + + sockets_.at(i) = socket; + } + + // Create l2cache + snprintf(sname, 100, "cluster%d-l2cache", cluster_id); l2cache_ = CacheSim::Create(sname, CacheSim::Config{ !L2_ENABLED, log2ceil(L2_CACHE_SIZE), // C - log2ceil(MEM_BLOCK_SIZE), // B + log2ceil(MEM_BLOCK_SIZE), // L log2ceil(L2_NUM_WAYS), // W 0, // A + log2ceil(L2_NUM_BANKS), // B XLEN, // address bits - L2_NUM_BANKS, // number of banks 1, // number of ports - 5, // request size + 2, // request size true, // write-through false, // write response - 0, // victim size L2_MSHR_SIZE, // mshr 2, // pipeline latency }); @@ -53,89 +79,11 @@ Cluster::Cluster(const SimContext& ctx, l2cache_->MemReqPort.bind(&this->mem_req_port); this->mem_rsp_port.bind(&l2cache_->MemRspPort); - snprintf(sname, 100, "cluster%d-icaches", cluster_id); - icaches_ = CacheCluster::Create(sname, num_cores, NUM_ICACHES, 1, CacheSim::Config{ - !ICACHE_ENABLED, - log2ceil(ICACHE_SIZE), // C - log2ceil(L1_LINE_SIZE), // B - log2ceil(sizeof(uint32_t)), // W - log2ceil(ICACHE_NUM_WAYS),// A - XLEN, // address bits - 1, // number of banks - 1, // number of ports - 1, // number of inputs - true, // write-through - false, // write response - 0, // victim size - (uint8_t)arch.num_warps(), // mshr - 2, // pipeline latency - }); + icache_switch->ReqOut.at(0).bind(&l2cache_->CoreReqPorts.at(0)); + l2cache_->CoreRspPorts.at(0).bind(&icache_switch->RspOut.at(0)); - icaches_->MemReqPort.bind(&l2cache_->CoreReqPorts.at(0)); - l2cache_->CoreRspPorts.at(0).bind(&icaches_->MemRspPort); - - snprintf(sname, 100, "cluster%d-dcaches", cluster_id); - dcaches_ = CacheCluster::Create(sname, num_cores, NUM_DCACHES, NUM_LSU_LANES, CacheSim::Config{ - !DCACHE_ENABLED, - log2ceil(DCACHE_SIZE), // C - log2ceil(L1_LINE_SIZE), // B - log2ceil(sizeof(Word)), // W - log2ceil(DCACHE_NUM_WAYS),// A - XLEN, // address bits - DCACHE_NUM_BANKS, // number of banks - 1, // number of ports - DCACHE_NUM_BANKS, // number of inputs - true, // write-through - false, // write response - 0, // victim size - DCACHE_MSHR_SIZE, // mshr - 4, // pipeline latency - }); - - dcaches_->MemReqPort.bind(&l2cache_->CoreReqPorts.at(1)); - l2cache_->CoreRspPorts.at(1).bind(&dcaches_->MemRspPort); - - /////////////////////////////////////////////////////////////////////////// - - // create shared memory blocks - for (uint32_t i = 0; i < num_cores; ++i) { - snprintf(sname, 100, "cluster%d-shared_mem%d", cluster_id, i); - sharedmems_.at(i) = SharedMem::Create(sname, SharedMem::Config{ - (1 << SMEM_LOG_SIZE), - sizeof(Word), - NUM_LSU_LANES, - NUM_LSU_LANES, - false - }); - } - - // create cores - - for (uint32_t i = 0; i < num_cores; ++i) { - uint32_t core_id = cluster_id * num_cores + i; - cores_.at(i) = Core::Create(core_id, - this, - arch, - dcrs, - sharedmems_.at(i)); - - cores_.at(i)->icache_req_ports.at(0).bind(&icaches_->CoreReqPorts.at(i).at(0)); - icaches_->CoreRspPorts.at(i).at(0).bind(&cores_.at(i)->icache_rsp_ports.at(0)); - - for (uint32_t j = 0; j < NUM_LSU_LANES; ++j) { - snprintf(sname, 100, "cluster%d-smem_demux%d_%d", cluster_id, i, j); - auto smem_demux = SMemDemux::Create(sname); - - cores_.at(i)->dcache_req_ports.at(j).bind(&smem_demux->ReqIn); - smem_demux->RspIn.bind(&cores_.at(i)->dcache_rsp_ports.at(j)); - - smem_demux->ReqDc.bind(&dcaches_->CoreReqPorts.at(i).at(j)); - dcaches_->CoreRspPorts.at(i).at(j).bind(&smem_demux->RspDc); - - smem_demux->ReqSm.bind(&sharedmems_.at(i)->Inputs.at(j)); - sharedmems_.at(i)->Outputs.at(j).bind(&smem_demux->RspSm); - } - } + dcache_switch->ReqOut.at(0).bind(&l2cache_->CoreReqPorts.at(1)); + l2cache_->CoreRspPorts.at(1).bind(&dcache_switch->RspOut.at(0)); } Cluster::~Cluster() { @@ -153,14 +101,14 @@ void Cluster::tick() { } void Cluster::attach_ram(RAM* ram) { - for (auto core : cores_) { - core->attach_ram(ram); + for (auto& socket : sockets_) { + socket->attach_ram(ram); } } bool Cluster::running() const { - for (auto& core : cores_) { - if (core->running()) + for (auto& socket : sockets_) { + if (socket->running()) return true; } return false; @@ -169,9 +117,9 @@ bool Cluster::running() const { bool Cluster::check_exit(Word* exitcode, bool riscv_test) const { bool done = true; Word exitcode_ = 0; - for (auto& core : cores_) { + for (auto& socket : sockets_) { Word ec; - if (core->check_exit(&ec, riscv_test)) { + if (socket->check_exit(&ec, riscv_test)) { exitcode_ |= ec; } else { done = false; @@ -184,36 +132,32 @@ bool Cluster::check_exit(Word* exitcode, bool riscv_test) const { void Cluster::barrier(uint32_t bar_id, uint32_t count, uint32_t core_id) { auto& barrier = barriers_.at(bar_id); - uint32_t local_core_id = core_id % cores_.size(); + auto sockets_per_cluster = sockets_.size(); + auto cores_per_socket = cores_per_socket_; + + uint32_t cores_per_cluster = sockets_per_cluster * cores_per_socket; + uint32_t local_core_id = core_id % cores_per_cluster; barrier.set(local_core_id); DP(3, "*** Suspend core #" << core_id << " at barrier #" << bar_id); if (barrier.count() == (size_t)count) { // resume all suspended cores - for (uint32_t i = 0; i < cores_.size(); ++i) { - if (barrier.test(i)) { - DP(3, "*** Resume core #" << i << " at barrier #" << bar_id); - cores_.at(i)->resume(); + for (uint32_t s = 0; s < sockets_per_cluster; ++s) { + for (uint32_t c = 0; c < cores_per_socket; ++c) { + uint32_t i = s * cores_per_socket + c; + if (barrier.test(i)) { + DP(3, "*** Resume core #" << i << " at barrier #" << bar_id); + sockets_.at(s)->resume(c); + } } } barrier.reset(); } } -ProcessorImpl* Cluster::processor() const { - return processor_; -} - Cluster::PerfStats Cluster::perf_stats() const { - Cluster::PerfStats perf; - perf.icache = icaches_->perf_stats(); - perf.dcache = dcaches_->perf_stats(); - perf.l2cache = l2cache_->perf_stats(); - - for (auto sharedmem : sharedmems_) { - perf.sharedmem += sharedmem->perf_stats(); - } - - return perf; + PerfStats perf_stats; + perf_stats.l2cache = l2cache_->perf_stats(); + return perf_stats; } \ No newline at end of file diff --git a/sim/simx/cluster.h b/sim/simx/cluster.h index f91241e9..81e93e10 100644 --- a/sim/simx/cluster.h +++ b/sim/simx/cluster.h @@ -19,6 +19,7 @@ #include "cache_cluster.h" #include "shared_mem.h" #include "core.h" +#include "socket.h" #include "constants.h" namespace vortex { @@ -28,18 +29,7 @@ class ProcessorImpl; class Cluster : public SimObject { public: struct PerfStats { - CacheSim::PerfStats icache; - CacheSim::PerfStats dcache; - SharedMem::PerfStats sharedmem; - CacheSim::PerfStats l2cache; - - PerfStats& operator+=(const PerfStats& rhs) { - this->icache += rhs.icache; - this->dcache += rhs.dcache; - this->sharedmem += rhs.sharedmem; - this->l2cache += rhs.l2cache; - return *this; - } + CacheSim::PerfStats l2cache; }; SimPort mem_req_port; @@ -53,6 +43,14 @@ public: ~Cluster(); + uint32_t id() const { + return cluster_id_; + } + + ProcessorImpl* processor() const { + return processor_; + } + void reset(); void tick(); @@ -65,22 +63,15 @@ public: void barrier(uint32_t bar_id, uint32_t count, uint32_t core_id); - ProcessorImpl* processor() const; - - Cluster::PerfStats perf_stats() const; + PerfStats perf_stats() const; private: - uint32_t cluster_id_; - std::vector cores_; - std::vector barriers_; - CacheSim::Ptr l2cache_; - CacheCluster::Ptr icaches_; - CacheCluster::Ptr dcaches_; - std::vector sharedmems_; - CacheCluster::Ptr tcaches_; - CacheCluster::Ptr ocaches_; - CacheCluster::Ptr rcaches_; - ProcessorImpl* processor_; + uint32_t cluster_id_; + ProcessorImpl* processor_; + std::vector sockets_; + std::vector barriers_; + CacheSim::Ptr l2cache_; + uint32_t cores_per_socket_; }; } // namespace vortex \ No newline at end of file diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index f2931324..1c155011 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -21,6 +21,7 @@ #include "mem.h" #include "decode.h" #include "core.h" +#include "socket.h" #include "debug.h" #include "constants.h" #include "processor_impl.h" @@ -29,35 +30,36 @@ using namespace vortex; Core::Core(const SimContext& ctx, uint32_t core_id, - Cluster* cluster, + Socket* socket, const Arch &arch, - const DCRS &dcrs, - SharedMem::Ptr sharedmem) + const DCRS &dcrs) : SimObject(ctx, "core") , icache_req_ports(1, this) , icache_rsp_ports(1, this) , dcache_req_ports(NUM_LSU_LANES, this) , dcache_rsp_ports(NUM_LSU_LANES, this) , core_id_(core_id) + , socket_(socket) , arch_(arch) , dcrs_(dcrs) , decoder_(arch) , warps_(arch.num_warps()) , barriers_(arch.num_barriers(), 0) , fcsrs_(arch.num_warps(), 0) - , ibuffers_(ISSUE_WIDTH, IBUF_SIZE) - , scoreboard_(arch_) + , ibuffers_(arch.num_warps(), IBUF_SIZE) + , scoreboard_(arch_) , operands_(ISSUE_WIDTH) - , dispatchers_((uint32_t)ExeType::MAX) - , exe_units_((uint32_t)ExeType::MAX) - , sharedmem_(sharedmem) + , dispatchers_((uint32_t)ExeType::ExeTypeCount) + , exe_units_((uint32_t)ExeType::ExeTypeCount) + , smem_demuxs_(NUM_LSU_LANES) , fetch_latch_("fetch") , decode_latch_("decode") , pending_icache_(arch_.num_warps()) - , committed_traces_(ISSUE_WIDTH, nullptr) - , csrs_(arch.num_warps()) - , cluster_(cluster) -{ + , csrs_(arch.num_warps()) + , commit_arbs_(ISSUE_WIDTH) +{ + char sname[100]; + for (uint32_t i = 0; i < arch_.num_warps(); ++i) { csrs_.at(i).resize(arch.num_threads()); } @@ -70,6 +72,28 @@ Core::Core(const SimContext& ctx, operands_.at(i) = SimPlatform::instance().create_object(); } + // initialize shared memory + snprintf(sname, 100, "core%d-shared_mem", core_id); + shared_mem_ = SharedMem::Create(sname, SharedMem::Config{ + (1 << SMEM_LOG_SIZE), + sizeof(Word), + NUM_LSU_LANES, + NUM_LSU_LANES, + false + }); + for (uint32_t i = 0; i < NUM_LSU_LANES; ++i) { + snprintf(sname, 100, "core%d-smem_demux%d", core_id, i); + auto smem_demux = SMemDemux::Create(sname); + + smem_demux->ReqDC.bind(&dcache_req_ports.at(i)); + dcache_rsp_ports.at(i).bind(&smem_demux->RspDC); + + smem_demux->ReqSM.bind(&shared_mem_->Inputs.at(i)); + shared_mem_->Outputs.at(i).bind(&smem_demux->RspSM); + + smem_demuxs_.at(i) = smem_demux; + } + // initialize dispatchers dispatchers_.at((int)ExeType::ALU) = SimPlatform::instance().create_object(arch, 2, NUM_ALU_BLOCKS, NUM_ALU_LANES); dispatchers_.at((int)ExeType::FPU) = SimPlatform::instance().create_object(arch, 2, NUM_FPU_BLOCKS, NUM_FPU_LANES); @@ -82,6 +106,16 @@ Core::Core(const SimContext& ctx, exe_units_.at((int)ExeType::LSU) = SimPlatform::instance().create_object(this); exe_units_.at((int)ExeType::SFU) = SimPlatform::instance().create_object(this); + // bind commit arbiters + for (uint32_t i = 0; i < ISSUE_WIDTH; ++i) { + snprintf(sname, 100, "core%d-commit-arb%d", core_id, i); + auto arbiter = TraceSwitch::Create(sname, ArbiterType::RoundRobin, (uint32_t)ExeType::ExeTypeCount, 1); + for (uint32_t j = 0; j < (uint32_t)ExeType::ExeTypeCount; ++j) { + exe_units_.at(j)->Outputs.at(i).bind(&arbiter->Inputs.at(j)); + } + commit_arbs_.at(i) = arbiter; + } + this->reset(); } @@ -99,8 +133,12 @@ void Core::reset() { for (auto& exe_unit : exe_units_) { exe_unit->reset(); } + + for (auto& commit_arb : commit_arbs_) { + commit_arb->reset(); + } - for ( auto& barrier : barriers_) { + for (auto& barrier : barriers_) { barrier.reset(); } @@ -112,7 +150,7 @@ void Core::reset() { ibuf.clear(); } - commit_exe_= 0; + ibuffer_idx_ = 0; scoreboard_.clear(); fetch_latch_.clear(); @@ -150,8 +188,10 @@ void Core::schedule() { break; } } - if (scheduled_warp == -1) + if (scheduled_warp == -1) { + ++perf_stats_.sched_idle; return; + } // suspend warp until decode stalled_warps_.set(scheduled_warp); @@ -192,11 +232,11 @@ void Core::fetch() { mem_req.tag = pending_icache_.allocate(trace); mem_req.cid = trace->cid; mem_req.uuid = trace->uuid; - icache_req_ports.at(0).send(mem_req, 1); + icache_req_ports.at(0).send(mem_req, 2); DT(3, "icache-req: addr=0x" << std::hex << mem_req.addr << ", tag=" << mem_req.tag << ", " << *trace); - fetch_latch_.pop(); - ++pending_ifetches_; + fetch_latch_.pop(); ++perf_stats_.ifetches; + ++pending_ifetches_; } void Core::decode() { @@ -206,7 +246,7 @@ void Core::decode() { auto trace = decode_latch_.front(); // check ibuffer capacity - auto& ibuffer = ibuffers_.at(trace->wid % ISSUE_WIDTH); + auto& ibuffer = ibuffers_.at(trace->wid); if (ibuffer.full()) { if (!trace->log_once(true)) { DT(3, "*** ibuffer-stall: " << *trace); @@ -223,13 +263,6 @@ void Core::decode() { stalled_warps_.reset(trace->wid); } - // update perf counters - uint32_t active_threads = trace->tmask.count(); - if (trace->exe_type == ExeType::LSU && trace->lsu_type == LsuType::LOAD) - perf_stats_.loads += active_threads; - if (trace->exe_type == ExeType::LSU && trace->lsu_type == LsuType::STORE) - perf_stats_.stores += active_threads; - DT(3, "pipeline-decode: " << *trace); // insert to ibuffer @@ -239,7 +272,7 @@ void Core::decode() { } void Core::issue() { - // operands to dispatch + // operands to dispatchers for (uint32_t i = 0; i < ISSUE_WIDTH; ++i) { auto& operand = operands_.at(i); if (operand->Output.empty()) @@ -257,7 +290,8 @@ void Core::issue() { // issue ibuffer instructions for (uint32_t i = 0; i < ISSUE_WIDTH; ++i) { - auto& ibuffer = ibuffers_.at(i); + uint32_t ii = (ibuffer_idx_ + i) % ibuffers_.size(); + auto& ibuffer = ibuffers_.at(ii); if (ibuffer.empty()) continue; @@ -265,17 +299,41 @@ void Core::issue() { // check scoreboard if (scoreboard_.in_use(trace)) { + auto uses = scoreboard_.get_uses(trace); if (!trace->log_once(true)) { - DTH(3, "*** scoreboard-stall: dependents={"); - auto uses = scoreboard_.get_uses(trace); + DTH(3, "*** scoreboard-stall: dependents={"); for (uint32_t j = 0, n = uses.size(); j < n; ++j) { auto& use = uses.at(j); __unused (use); if (j) DTN(3, ", "); - DTN(3, use.type << use.reg << "(#" << use.owner << ")"); + DTN(3, use.reg_type << use.reg_id << "(#" << use.uuid << ")"); } DTN(3, "}, " << *trace << std::endl); } + for (uint32_t j = 0, n = uses.size(); j < n; ++j) { + auto& use = uses.at(j); + switch (use.exe_type) { + case ExeType::ALU: ++perf_stats_.scrb_alu; break; + case ExeType::FPU: ++perf_stats_.scrb_fpu; break; + case ExeType::LSU: ++perf_stats_.scrb_lsu; break; + case ExeType::SFU: { + ++perf_stats_.scrb_sfu; + switch (use.sfu_type) { + case SfuType::TMC: + case SfuType::WSPAWN: + case SfuType::SPLIT: + case SfuType::JOIN: + case SfuType::BAR: + case SfuType::PRED: ++perf_stats_.scrb_wctl; break; + case SfuType::CSRRW: + case SfuType::CSRRS: + case SfuType::CSRRC: ++perf_stats_.scrb_csrs; break; + default: assert(false); + } + } break; + default: assert(false); + } + } ++perf_stats_.scrb_stalls; continue; } else { @@ -294,10 +352,11 @@ void Core::issue() { ibuffer.pop(); } + ibuffer_idx_ += ISSUE_WIDTH; } void Core::execute() { - for (uint32_t i = 0; i < (uint32_t)ExeType::MAX; ++i) { + for (uint32_t i = 0; i < (uint32_t)ExeType::ExeTypeCount; ++i) { auto& dispatch = dispatchers_.at(i); auto& exe_unit = exe_units_.at(i); for (uint32_t j = 0; j < ISSUE_WIDTH; ++j) { @@ -313,10 +372,10 @@ void Core::execute() { void Core::commit() { // process completed instructions for (uint32_t i = 0; i < ISSUE_WIDTH; ++i) { - auto trace = committed_traces_.at(i); - if (!trace) + auto& commit_arb = commit_arbs_.at(i); + if (commit_arb->Outputs.at(0).empty()) continue; - committed_traces_.at(i) = nullptr; + auto trace = commit_arb->Outputs.at(0).front(); // advance to commit stage DT(3, "pipeline-commit: " << *trace); @@ -334,27 +393,11 @@ void Core::commit() { perf_stats_.instrs += trace->tmask.count(); } + commit_arb->Outputs.at(0).pop(); + // delete the trace delete trace; } - - // select completed instructions - for (uint32_t i = 0; i < (uint32_t)ExeType::MAX; ++i) { - uint32_t ii = (commit_exe_ + i) % (uint32_t)ExeType::MAX; - auto& exe_unit = exe_units_.at(ii); - for (uint32_t j = 0; j < ISSUE_WIDTH; ++j) { - auto committed_trace = committed_traces_.at(j); - if (committed_trace) - continue; - auto& output = exe_unit->Outputs.at(j); - if (output.empty()) - continue; - auto trace = output.front(); - committed_traces_.at(j) = trace; - output.pop(); - } - } - ++commit_exe_; } void Core::wspawn(uint32_t num_warps, Word nextPC) { @@ -379,7 +422,7 @@ void Core::barrier(uint32_t bar_id, uint32_t count, uint32_t warp_id) { if (is_global) { // global barrier handling if (barrier.count() == active_warps_.count()) { - cluster_->barrier(bar_idx, count, core_id_); + socket_->barrier(bar_idx, count, core_id_); barrier.reset(); } } else { @@ -416,7 +459,7 @@ AddrType Core::get_addr_type(uint64_t addr) { void Core::dcache_read(void *data, uint64_t addr, uint32_t size) { auto type = this->get_addr_type(addr); if (type == AddrType::Shared) { - sharedmem_->read(data, addr, size); + shared_mem_->read(data, addr, size); } else { mmu_.read(data, addr, size, 0); } @@ -431,7 +474,7 @@ void Core::dcache_write(const void* data, uint64_t addr, uint32_t size) { this->writeToStdOut(data, addr, size); } else { if (type == AddrType::Shared) { - sharedmem_->write(data, addr, size); + shared_mem_->write(data, addr, size); } else { mmu_.write(data, addr, size, 0); } @@ -533,71 +576,76 @@ uint32_t Core::get_csr(uint32_t addr, uint32_t tid, uint32_t wid) { break; case VX_DCR_MPM_CLASS_CORE: { switch (addr) { + case VX_CSR_MPM_SCHED_ID: return perf_stats_.sched_idle & 0xffffffff; + case VX_CSR_MPM_SCHED_ID_H:return perf_stats_.sched_idle >> 32; + case VX_CSR_MPM_SCHED_ST: return perf_stats_.sched_stalls & 0xffffffff; + case VX_CSR_MPM_SCHED_ST_H:return perf_stats_.sched_stalls >> 32; case VX_CSR_MPM_IBUF_ST: return perf_stats_.ibuf_stalls & 0xffffffff; case VX_CSR_MPM_IBUF_ST_H: return perf_stats_.ibuf_stalls >> 32; - case VX_CSR_MPM_SCRB_ST: return perf_stats_.scrb_stalls & 0xffffffff; - case VX_CSR_MPM_SCRB_ST_H: return perf_stats_.scrb_stalls >> 32; - case VX_CSR_MPM_ALU_ST: return perf_stats_.alu_stalls & 0xffffffff; - case VX_CSR_MPM_ALU_ST_H: return perf_stats_.alu_stalls >> 32; - case VX_CSR_MPM_LSU_ST: return perf_stats_.lsu_stalls & 0xffffffff; - case VX_CSR_MPM_LSU_ST_H: return perf_stats_.lsu_stalls >> 32; - case VX_CSR_MPM_FPU_ST: return perf_stats_.fpu_stalls & 0xffffffff; - case VX_CSR_MPM_FPU_ST_H: return perf_stats_.fpu_stalls >> 32; - case VX_CSR_MPM_SFU_ST: return perf_stats_.sfu_stalls & 0xffffffff; - case VX_CSR_MPM_SFU_ST_H: return perf_stats_.sfu_stalls >> 32; - + case VX_CSR_MPM_SCRB_ST: return perf_stats_.scrb_stalls & 0xffffffff; + case VX_CSR_MPM_SCRB_ST_H: return perf_stats_.scrb_stalls >> 32; + case VX_CSR_MPM_SCRB_ALU: return perf_stats_.scrb_alu & 0xffffffff; + case VX_CSR_MPM_SCRB_ALU_H:return perf_stats_.scrb_alu >> 32; + case VX_CSR_MPM_SCRB_FPU: return perf_stats_.scrb_fpu & 0xffffffff; + case VX_CSR_MPM_SCRB_FPU_H:return perf_stats_.scrb_fpu >> 32; + case VX_CSR_MPM_SCRB_LSU: return perf_stats_.scrb_lsu & 0xffffffff; + case VX_CSR_MPM_SCRB_LSU_H:return perf_stats_.scrb_lsu >> 32; + case VX_CSR_MPM_SCRB_SFU: return perf_stats_.scrb_sfu & 0xffffffff; + case VX_CSR_MPM_SCRB_SFU_H:return perf_stats_.scrb_sfu >> 32; + case VX_CSR_MPM_SCRB_WCTL: return perf_stats_.scrb_wctl & 0xffffffff; + case VX_CSR_MPM_SCRB_WCTL_H: return perf_stats_.scrb_wctl >> 32; + case VX_CSR_MPM_SCRB_CSRS: return perf_stats_.scrb_csrs & 0xffffffff; + case VX_CSR_MPM_SCRB_CSRS_H: return perf_stats_.scrb_csrs >> 32; case VX_CSR_MPM_IFETCHES: return perf_stats_.ifetches & 0xffffffff; case VX_CSR_MPM_IFETCHES_H: return perf_stats_.ifetches >> 32; case VX_CSR_MPM_LOADS: return perf_stats_.loads & 0xffffffff; case VX_CSR_MPM_LOADS_H: return perf_stats_.loads >> 32; case VX_CSR_MPM_STORES: return perf_stats_.stores & 0xffffffff; case VX_CSR_MPM_STORES_H: return perf_stats_.stores >> 32; - case VX_CSR_MPM_IFETCH_LAT: return perf_stats_.ifetch_latency & 0xffffffff; - case VX_CSR_MPM_IFETCH_LAT_H: return perf_stats_.ifetch_latency >> 32; - case VX_CSR_MPM_LOAD_LAT: return perf_stats_.load_latency & 0xffffffff; - case VX_CSR_MPM_LOAD_LAT_H: return perf_stats_.load_latency >> 32; + case VX_CSR_MPM_IFETCH_LT: return perf_stats_.ifetch_latency & 0xffffffff; + case VX_CSR_MPM_IFETCH_LT_H: return perf_stats_.ifetch_latency >> 32; + case VX_CSR_MPM_LOAD_LT: return perf_stats_.load_latency & 0xffffffff; + case VX_CSR_MPM_LOAD_LT_H: return perf_stats_.load_latency >> 32; } } break; case VX_DCR_MPM_CLASS_MEM: { - auto proc_perf = cluster_->processor()->perf_stats(); + auto proc_perf = socket_->cluster()->processor()->perf_stats(); + auto cluster_perf = socket_->cluster()->perf_stats(); + auto socket_perf = socket_->perf_stats(); + auto smem_perf = shared_mem_->perf_stats(); switch (addr) { - case VX_CSR_MPM_ICACHE_READS: return proc_perf.clusters.icache.reads & 0xffffffff; - case VX_CSR_MPM_ICACHE_READS_H: return proc_perf.clusters.icache.reads >> 32; - case VX_CSR_MPM_ICACHE_MISS_R: return proc_perf.clusters.icache.read_misses & 0xffffffff; - case VX_CSR_MPM_ICACHE_MISS_R_H: return proc_perf.clusters.icache.read_misses >> 32; + case VX_CSR_MPM_ICACHE_READS: return socket_perf.icache.reads & 0xffffffff; + case VX_CSR_MPM_ICACHE_READS_H: return socket_perf.icache.reads >> 32; + case VX_CSR_MPM_ICACHE_MISS_R: return socket_perf.icache.read_misses & 0xffffffff; + case VX_CSR_MPM_ICACHE_MISS_R_H: return socket_perf.icache.read_misses >> 32; + case VX_CSR_MPM_ICACHE_MSHR_ST: return socket_perf.icache.mshr_stalls & 0xffffffff; + case VX_CSR_MPM_ICACHE_MSHR_ST_H: return socket_perf.icache.mshr_stalls >> 32; - case VX_CSR_MPM_DCACHE_READS: return proc_perf.clusters.dcache.reads & 0xffffffff; - case VX_CSR_MPM_DCACHE_READS_H: return proc_perf.clusters.dcache.reads >> 32; - case VX_CSR_MPM_DCACHE_WRITES: return proc_perf.clusters.dcache.writes & 0xffffffff; - case VX_CSR_MPM_DCACHE_WRITES_H: return proc_perf.clusters.dcache.writes >> 32; - case VX_CSR_MPM_DCACHE_MISS_R: return proc_perf.clusters.dcache.read_misses & 0xffffffff; - case VX_CSR_MPM_DCACHE_MISS_R_H: return proc_perf.clusters.dcache.read_misses >> 32; - case VX_CSR_MPM_DCACHE_MISS_W: return proc_perf.clusters.dcache.write_misses & 0xffffffff; - case VX_CSR_MPM_DCACHE_MISS_W_H: return proc_perf.clusters.dcache.write_misses >> 32; - case VX_CSR_MPM_DCACHE_BANK_ST: return proc_perf.clusters.dcache.bank_stalls & 0xffffffff; - case VX_CSR_MPM_DCACHE_BANK_ST_H:return proc_perf.clusters.dcache.bank_stalls >> 32; - case VX_CSR_MPM_DCACHE_MSHR_ST: return proc_perf.clusters.dcache.mshr_stalls & 0xffffffff; - case VX_CSR_MPM_DCACHE_MSHR_ST_H:return proc_perf.clusters.dcache.mshr_stalls >> 32; - - case VX_CSR_MPM_SMEM_READS: return proc_perf.clusters.sharedmem.reads & 0xffffffff; - case VX_CSR_MPM_SMEM_READS_H: return proc_perf.clusters.sharedmem.reads >> 32; - case VX_CSR_MPM_SMEM_WRITES: return proc_perf.clusters.sharedmem.writes & 0xffffffff; - case VX_CSR_MPM_SMEM_WRITES_H: return proc_perf.clusters.sharedmem.writes >> 32; - case VX_CSR_MPM_SMEM_BANK_ST: return proc_perf.clusters.sharedmem.bank_stalls & 0xffffffff; - case VX_CSR_MPM_SMEM_BANK_ST_H:return proc_perf.clusters.sharedmem.bank_stalls >> 32; + case VX_CSR_MPM_DCACHE_READS: return socket_perf.dcache.reads & 0xffffffff; + case VX_CSR_MPM_DCACHE_READS_H: return socket_perf.dcache.reads >> 32; + case VX_CSR_MPM_DCACHE_WRITES: return socket_perf.dcache.writes & 0xffffffff; + case VX_CSR_MPM_DCACHE_WRITES_H: return socket_perf.dcache.writes >> 32; + case VX_CSR_MPM_DCACHE_MISS_R: return socket_perf.dcache.read_misses & 0xffffffff; + case VX_CSR_MPM_DCACHE_MISS_R_H: return socket_perf.dcache.read_misses >> 32; + case VX_CSR_MPM_DCACHE_MISS_W: return socket_perf.dcache.write_misses & 0xffffffff; + case VX_CSR_MPM_DCACHE_MISS_W_H: return socket_perf.dcache.write_misses >> 32; + case VX_CSR_MPM_DCACHE_BANK_ST: return socket_perf.dcache.bank_stalls & 0xffffffff; + case VX_CSR_MPM_DCACHE_BANK_ST_H: return socket_perf.dcache.bank_stalls >> 32; + case VX_CSR_MPM_DCACHE_MSHR_ST: return socket_perf.dcache.mshr_stalls & 0xffffffff; + case VX_CSR_MPM_DCACHE_MSHR_ST_H: return socket_perf.dcache.mshr_stalls >> 32; - case VX_CSR_MPM_L2CACHE_READS: return proc_perf.clusters.l2cache.reads & 0xffffffff; - case VX_CSR_MPM_L2CACHE_READS_H: return proc_perf.clusters.l2cache.reads >> 32; - case VX_CSR_MPM_L2CACHE_WRITES: return proc_perf.clusters.l2cache.writes & 0xffffffff; - case VX_CSR_MPM_L2CACHE_WRITES_H: return proc_perf.clusters.l2cache.writes >> 32; - case VX_CSR_MPM_L2CACHE_MISS_R: return proc_perf.clusters.l2cache.read_misses & 0xffffffff; - case VX_CSR_MPM_L2CACHE_MISS_R_H: return proc_perf.clusters.l2cache.read_misses >> 32; - case VX_CSR_MPM_L2CACHE_MISS_W: return proc_perf.clusters.l2cache.write_misses & 0xffffffff; - case VX_CSR_MPM_L2CACHE_MISS_W_H: return proc_perf.clusters.l2cache.write_misses >> 32; - case VX_CSR_MPM_L2CACHE_BANK_ST: return proc_perf.clusters.l2cache.bank_stalls & 0xffffffff; - case VX_CSR_MPM_L2CACHE_BANK_ST_H:return proc_perf.clusters.l2cache.bank_stalls >> 32; - case VX_CSR_MPM_L2CACHE_MSHR_ST: return proc_perf.clusters.l2cache.mshr_stalls & 0xffffffff; - case VX_CSR_MPM_L2CACHE_MSHR_ST_H:return proc_perf.clusters.l2cache.mshr_stalls >> 32; + case VX_CSR_MPM_L2CACHE_READS: return cluster_perf.l2cache.reads & 0xffffffff; + case VX_CSR_MPM_L2CACHE_READS_H: return cluster_perf.l2cache.reads >> 32; + case VX_CSR_MPM_L2CACHE_WRITES: return cluster_perf.l2cache.writes & 0xffffffff; + case VX_CSR_MPM_L2CACHE_WRITES_H: return cluster_perf.l2cache.writes >> 32; + case VX_CSR_MPM_L2CACHE_MISS_R: return cluster_perf.l2cache.read_misses & 0xffffffff; + case VX_CSR_MPM_L2CACHE_MISS_R_H: return cluster_perf.l2cache.read_misses >> 32; + case VX_CSR_MPM_L2CACHE_MISS_W: return cluster_perf.l2cache.write_misses & 0xffffffff; + case VX_CSR_MPM_L2CACHE_MISS_W_H: return cluster_perf.l2cache.write_misses >> 32; + case VX_CSR_MPM_L2CACHE_BANK_ST: return cluster_perf.l2cache.bank_stalls & 0xffffffff; + case VX_CSR_MPM_L2CACHE_BANK_ST_H:return cluster_perf.l2cache.bank_stalls >> 32; + case VX_CSR_MPM_L2CACHE_MSHR_ST: return cluster_perf.l2cache.mshr_stalls & 0xffffffff; + case VX_CSR_MPM_L2CACHE_MSHR_ST_H:return cluster_perf.l2cache.mshr_stalls >> 32; case VX_CSR_MPM_L3CACHE_READS: return proc_perf.l3cache.reads & 0xffffffff; case VX_CSR_MPM_L3CACHE_READS_H: return proc_perf.l3cache.reads >> 32; @@ -612,14 +660,25 @@ uint32_t Core::get_csr(uint32_t addr, uint32_t tid, uint32_t wid) { case VX_CSR_MPM_L3CACHE_MSHR_ST: return proc_perf.l3cache.mshr_stalls & 0xffffffff; case VX_CSR_MPM_L3CACHE_MSHR_ST_H:return proc_perf.l3cache.mshr_stalls >> 32; - case VX_CSR_MPM_MEM_READS: return proc_perf.mem_reads & 0xffffffff; - case VX_CSR_MPM_MEM_READS_H: return proc_perf.mem_reads >> 32; - case VX_CSR_MPM_MEM_WRITES: return proc_perf.mem_writes & 0xffffffff; - case VX_CSR_MPM_MEM_WRITES_H:return proc_perf.mem_writes >> 32; - case VX_CSR_MPM_MEM_LAT: return proc_perf.mem_latency & 0xffffffff; - case VX_CSR_MPM_MEM_LAT_H: return proc_perf.mem_latency >> 32; + case VX_CSR_MPM_MEM_READS: return proc_perf.mem_reads & 0xffffffff; + case VX_CSR_MPM_MEM_READS_H: return proc_perf.mem_reads >> 32; + case VX_CSR_MPM_MEM_WRITES: return proc_perf.mem_writes & 0xffffffff; + case VX_CSR_MPM_MEM_WRITES_H: return proc_perf.mem_writes >> 32; + case VX_CSR_MPM_MEM_LT: return proc_perf.mem_latency & 0xffffffff; + case VX_CSR_MPM_MEM_LT_H : return proc_perf.mem_latency >> 32; + + case VX_CSR_MPM_SMEM_READS: return smem_perf.reads & 0xffffffff; + case VX_CSR_MPM_SMEM_READS_H: return smem_perf.reads >> 32; + case VX_CSR_MPM_SMEM_WRITES: return smem_perf.writes & 0xffffffff; + case VX_CSR_MPM_SMEM_WRITES_H: return smem_perf.writes >> 32; + case VX_CSR_MPM_SMEM_BANK_ST: return smem_perf.bank_stalls & 0xffffffff; + case VX_CSR_MPM_SMEM_BANK_ST_H: return smem_perf.bank_stalls >> 32; } } break; + default: { + std::cout << std::dec << "Error: invalid MPM CLASS: value=" << perf_class << std::endl; + std::abort(); + } break; } } else { std::cout << std::hex << "Error: invalid CSR read addr=0x" << addr << std::endl; diff --git a/sim/simx/core.h b/sim/simx/core.h index ed06574d..0ccb5d02 100644 --- a/sim/simx/core.h +++ b/sim/simx/core.h @@ -22,11 +22,11 @@ #include #include #include +#include #include "debug.h" #include "types.h" #include "arch.h" #include "decode.h" -#include "mem.h" #include "warp.h" #include "pipeline.h" #include "cache_sim.h" @@ -40,19 +40,25 @@ namespace vortex { -class Cluster; +class Socket; + +using TraceSwitch = Mux; class Core : public SimObject { public: struct PerfStats { uint64_t cycles; uint64_t instrs; + uint64_t sched_idle; + uint64_t sched_stalls; uint64_t ibuf_stalls; uint64_t scrb_stalls; - uint64_t alu_stalls; - uint64_t lsu_stalls; - uint64_t fpu_stalls; - uint64_t sfu_stalls; + uint64_t scrb_alu; + uint64_t scrb_fpu; + uint64_t scrb_lsu; + uint64_t scrb_sfu; + uint64_t scrb_wctl; + uint64_t scrb_csrs; uint64_t ifetches; uint64_t loads; uint64_t stores; @@ -62,12 +68,16 @@ public: PerfStats() : cycles(0) , instrs(0) + , sched_idle(0) + , sched_stalls(0) , ibuf_stalls(0) , scrb_stalls(0) - , alu_stalls(0) - , lsu_stalls(0) - , fpu_stalls(0) - , sfu_stalls(0) + , scrb_alu(0) + , scrb_fpu(0) + , scrb_lsu(0) + , scrb_sfu(0) + , scrb_wctl(0) + , scrb_csrs(0) , ifetches(0) , loads(0) , stores(0) @@ -84,10 +94,9 @@ public: Core(const SimContext& ctx, uint32_t core_id, - Cluster* cluster, + Socket* socket, const Arch &arch, - const DCRS &dcrs, - SharedMem::Ptr sharedmem); + const DCRS &dcrs); ~Core(); @@ -105,6 +114,10 @@ public: return core_id_; } + Socket* socket() const { + return socket_; + } + const Arch& arch() const { return arch_; } @@ -153,6 +166,7 @@ private: void cout_flush(); uint32_t core_id_; + Socket* socket_; const Arch& arch_; const DCRS &dcrs_; @@ -167,13 +181,13 @@ private: std::vector operands_; std::vector dispatchers_; std::vector exe_units_; - SharedMem::Ptr sharedmem_; + SharedMem::Ptr shared_mem_; + std::vector smem_demuxs_; PipelineLatch fetch_latch_; PipelineLatch decode_latch_; HashTable pending_icache_; - std::vector committed_traces_; WarpMask active_warps_; WarpMask stalled_warps_; uint64_t issued_instrs_; @@ -188,9 +202,10 @@ private: PerfStats perf_stats_; - Cluster* cluster_; + std::vector commit_arbs_; uint32_t commit_exe_; + uint32_t ibuffer_idx_; friend class Warp; friend class LsuUnit; diff --git a/sim/simx/dispatcher.h b/sim/simx/dispatcher.h index a5c44b64..fe83e9de 100644 --- a/sim/simx/dispatcher.h +++ b/sim/simx/dispatcher.h @@ -66,6 +66,7 @@ public: } auto& output = Outputs.at(i); auto trace = input.front(); + auto new_trace = trace; if (pid_count_ != 1) { auto start_p = start_p_.at(b); if (start_p == -1) { @@ -81,33 +82,30 @@ public: end = j; } start /= num_lanes_; - end /= num_lanes_; - auto new_trace = new pipeline_trace_t(*trace); - new_trace->tmask.reset(); - for (int j = start * num_lanes_, n = j + num_lanes_; j < n; ++j) { - new_trace->tmask[j] = trace->tmask[j]; - } - new_trace->pid = start; - new_trace->sop = (start_p == 0); - if (start == end) { - new_trace->eop = 1; + end /= num_lanes_; + if (start != end) { + new_trace = new pipeline_trace_t(*trace); + new_trace->eop = false; + start_p_.at(b) = start + 1; + } else { start_p_.at(b) = -1; input.pop(); ++block_sent; - delete trace; - } else { - new_trace->eop = 0; - start_p_.at(b) = start + 1; - } - output.send(new_trace, 1); - DT(3, "pipeline-dispatch: " << *new_trace); + } + new_trace->pid = start; + new_trace->sop = (0 == start_p); + ThreadMask tmask; + for (int j = start * num_lanes_, n = j + num_lanes_; j < n; ++j) { + tmask[j] = trace->tmask[j]; + } + new_trace->tmask = tmask; } else { - trace->pid = 0; + new_trace->pid = 0; input.pop(); - output.send(trace, 1); - DT(3, "pipeline-dispatch: " << *trace); ++block_sent; - } + } + DT(3, "pipeline-dispatch: " << *new_trace); + output.send(new_trace, 1); } if (block_sent == block_size_) { batch_idx_ = (batch_idx_ + 1) % batch_count_; @@ -138,4 +136,4 @@ private: std::vector start_p_; }; -} \ No newline at end of file +} diff --git a/sim/simx/exe_unit.cpp b/sim/simx/exe_unit.cpp index 2f3e79e3..4b5cb356 100644 --- a/sim/simx/exe_unit.cpp +++ b/sim/simx/exe_unit.cpp @@ -51,8 +51,7 @@ void AluUnit::tick() { assert(core_->stalled_warps_.test(trace->wid)); core_->stalled_warps_.reset(trace->wid); } - auto time = input.pop(); - core_->perf_stats_.alu_stalls += (SimPlatform::instance().cycles() - time); + input.pop(); } } @@ -87,8 +86,7 @@ void FpuUnit::tick() { std::abort(); } DT(3, "pipeline-execute: op=" << trace->fpu_type << ", " << *trace); - auto time = input.pop(); - core_->perf_stats_.fpu_stalls += (SimPlatform::instance().cycles() - time); + input.pop(); } } @@ -114,7 +112,7 @@ void LsuUnit::tick() { // handle dcache response for (uint32_t t = 0; t < num_lanes_; ++t) { - auto& dcache_rsp_port = core_->dcache_rsp_ports.at(t); + auto& dcache_rsp_port = core_->smem_demuxs_.at(t)->RspIn; if (dcache_rsp_port.empty()) continue; auto& mem_rsp = dcache_rsp_port.front(); @@ -136,7 +134,7 @@ void LsuUnit::tick() { // handle shared memory response for (uint32_t t = 0; t < num_lanes_; ++t) { - auto& smem_rsp_port = core_->sharedmem_->Outputs.at(t); + auto& smem_rsp_port = core_->shared_mem_->Outputs.at(t); if (smem_rsp_port.empty()) continue; auto& mem_rsp = smem_rsp_port.front(); @@ -184,8 +182,7 @@ void LsuUnit::tick() { fence_lock_ = true; DT(3, "fence-lock: " << *trace); // remove input - auto time = input.pop(); - core_->perf_stats_.lsu_stalls += (SimPlatform::instance().cycles() - time); + input.pop(); break; } @@ -213,7 +210,9 @@ void LsuUnit::tick() { auto mem_addr = trace_data->mem_addrs.at(t).addr & ~addr_mask; matches += (addr0 == mem_addr); } + #ifdef LSU_DUP_ENABLE is_dup = (matches == trace->tmask.count()); + #endif } uint32_t addr_count; @@ -229,7 +228,7 @@ void LsuUnit::tick() { if (!trace->tmask.test(t0 + t)) continue; - auto& dcache_req_port = core_->dcache_req_ports.at(t); + auto& dcache_req_port = core_->smem_demuxs_.at(t)->ReqIn; auto mem_addr = trace_data->mem_addrs.at(t); auto type = core_->get_addr_type(mem_addr.addr); @@ -241,12 +240,16 @@ void LsuUnit::tick() { mem_req.cid = trace->cid; mem_req.uuid = trace->uuid; - dcache_req_port.send(mem_req, 2); + dcache_req_port.send(mem_req, 1); DT(3, "dcache-req: addr=0x" << std::hex << mem_req.addr << ", tag=" << tag << ", lsu_type=" << trace->lsu_type << ", tid=" << t << ", addr_type=" << mem_req.type << ", " << *trace); - ++pending_loads_; - ++core_->perf_stats_.loads; + if (is_write) { + ++core_->perf_stats_.stores; + } else { + ++core_->perf_stats_.loads; + ++pending_loads_; + } if (is_dup) break; } @@ -254,13 +257,11 @@ void LsuUnit::tick() { // do not wait on writes if (is_write) { pending_rd_reqs_.release(tag); - output.send(trace, 1); - ++core_->perf_stats_.stores; + output.send(trace, 1); } // remove input - auto time = input.pop(); - core_->perf_stats_.lsu_stalls += (SimPlatform::instance().cycles() - time); + input.pop(); break; // single block } @@ -318,10 +319,7 @@ void SfuUnit::tick() { core_->stalled_warps_.reset(trace->wid); } - auto time = input.pop(); - auto stalls = (SimPlatform::instance().cycles() - time); - - core_->perf_stats_.sfu_stalls += stalls; + input.pop(); break; // single block } diff --git a/sim/simx/main.cpp b/sim/simx/main.cpp index 22d9c880..64031bb8 100644 --- a/sim/simx/main.cpp +++ b/sim/simx/main.cpp @@ -34,14 +34,13 @@ static void show_usage() { uint32_t num_threads = NUM_THREADS; uint32_t num_warps = NUM_WARPS; uint32_t num_cores = NUM_CORES; -uint32_t num_clusters = NUM_CLUSTERS; bool showStats = false;; bool riscv_test = false; const char* program = nullptr; static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "t:w:c:g:rsh?")) != -1) { + while ((c = getopt(argc, argv, "t:w:c:rsh?")) != -1) { switch (c) { case 't': num_threads = atoi(optarg); @@ -51,9 +50,6 @@ static void parse_args(int argc, char **argv) { break; case 'c': num_cores = atoi(optarg); - break; - case 'g': - num_clusters = atoi(optarg); break; case 'r': riscv_test = true; @@ -88,7 +84,7 @@ int main(int argc, char **argv) { { // create processor configuation - Arch arch(num_threads, num_warps, num_cores, num_clusters); + Arch arch(num_threads, num_warps, num_cores); // create memory module RAM ram(RAM_PAGE_SIZE); diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index da151b62..8e8c1062 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -32,18 +32,17 @@ ProcessorImpl::ProcessorImpl(const Arch& arch) l3cache_ = CacheSim::Create("l3cache", CacheSim::Config{ !L3_ENABLED, log2ceil(L3_CACHE_SIZE), // C - log2ceil(MEM_BLOCK_SIZE), // B - log2ceil(L3_NUM_WAYS), // W - 0, // A - XLEN, // address bits - L3_NUM_BANKS, // number of banks - 1, // number of ports + log2ceil(MEM_BLOCK_SIZE), // L + log2ceil(L3_NUM_WAYS), // W + 0, // A + log2ceil(L3_NUM_BANKS), // B + XLEN, // address bits + 1, // number of ports uint8_t(arch.num_clusters()), // request size - true, // write-through - false, // write response - 0, // victim size - L3_MSHR_SIZE, // mshr - 2, // pipeline latency + true, // write-through + false, // write response + L3_MSHR_SIZE, // mshr + 2, // pipeline latency } ); @@ -114,6 +113,7 @@ void ProcessorImpl::reset() { perf_mem_writes_ = 0; perf_mem_latency_ = 0; perf_mem_pending_reads_ = 0; + } void ProcessorImpl::write_dcr(uint32_t addr, uint32_t value) { @@ -126,9 +126,6 @@ ProcessorImpl::PerfStats ProcessorImpl::perf_stats() const { perf.mem_writes = perf_mem_writes_; perf.mem_latency = perf_mem_latency_; perf.l3cache = l3cache_->perf_stats(); - for (auto cluster : clusters_) { - perf.clusters += cluster->perf_stats(); - } return perf; } diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index 02d92e95..072247a7 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -24,17 +24,10 @@ namespace vortex { class ProcessorImpl { public: struct PerfStats { + CacheSim::PerfStats l3cache; uint64_t mem_reads; uint64_t mem_writes; uint64_t mem_latency; - CacheSim::PerfStats l3cache; - Cluster::PerfStats clusters; - - PerfStats() - : mem_reads(0) - , mem_writes(0) - , mem_latency(0) - {} }; ProcessorImpl(const Arch& arch); @@ -46,7 +39,7 @@ public: void write_dcr(uint32_t addr, uint32_t value); - ProcessorImpl::PerfStats perf_stats() const; + PerfStats perf_stats() const; private: @@ -55,7 +48,7 @@ private: const Arch& arch_; std::vector> clusters_; DCRS dcrs_; - MemSim::Ptr memsim_; + MemSim::Ptr memsim_; CacheSim::Ptr l3cache_; uint64_t perf_mem_reads_; uint64_t perf_mem_writes_; diff --git a/sim/simx/scoreboard.h b/sim/simx/scoreboard.h index 4d311846..58dbc2fb 100644 --- a/sim/simx/scoreboard.h +++ b/sim/simx/scoreboard.h @@ -22,9 +22,11 @@ class Scoreboard { public: struct reg_use_t { - RegType type; - uint32_t reg; - uint64_t owner; + RegType reg_type; + uint32_t reg_id; + ExeType exe_type; + SfuType sfu_type; + uint64_t uuid; }; Scoreboard(const Arch &arch) @@ -44,89 +46,81 @@ public: owners_.clear(); } - bool in_use(pipeline_trace_t* state) const { - return (state->used_iregs & in_use_iregs_.at(state->wid)) != 0 - || (state->used_fregs & in_use_fregs_.at(state->wid)) != 0 - || (state->used_vregs & in_use_vregs_.at(state->wid)) != 0; + bool in_use(pipeline_trace_t* trace) const { + return (trace->used_iregs & in_use_iregs_.at(trace->wid)) != 0 + || (trace->used_fregs & in_use_fregs_.at(trace->wid)) != 0 + || (trace->used_vregs & in_use_vregs_.at(trace->wid)) != 0; } - std::vector get_uses(pipeline_trace_t* state) const { - std::vector out; - { - uint32_t r = 0; - auto used_iregs = state->used_iregs & in_use_iregs_.at(state->wid); - while (used_iregs.any()) { - if (used_iregs.test(0)) { - uint32_t tag = (r << 16) | (state->wid << 4) | (int)RegType::Integer; - out.push_back({RegType::Integer, r, owners_.at(tag)}); - } - used_iregs >>= 1; - ++r; + std::vector get_uses(pipeline_trace_t* trace) const { + std::vector out; + + auto used_iregs = trace->used_iregs & in_use_iregs_.at(trace->wid); + auto used_fregs = trace->used_fregs & in_use_fregs_.at(trace->wid); + auto used_vregs = trace->used_vregs & in_use_vregs_.at(trace->wid); + + for (uint32_t r = 0; r < MAX_NUM_REGS; ++r) { + if (used_iregs.test(r)) { + uint32_t tag = (r << 16) | (trace->wid << 4) | (int)RegType::Integer; + auto owner = owners_.at(tag); + out.push_back({RegType::Integer, r, owner->exe_type, owner->sfu_type, owner->uuid}); } } - { - uint32_t r = 0; - auto used_fregs = state->used_fregs & in_use_fregs_.at(state->wid); - while (used_fregs.any()) { - if (used_fregs.test(0)) { - uint32_t tag = (r << 16) | (state->wid << 4) | (int)RegType::Float; - out.push_back({RegType::Float, r, owners_.at(tag)}); - } - used_fregs >>= 1; - ++r; + + for (uint32_t r = 0; r < MAX_NUM_REGS; ++r) { + if (used_fregs.test(r)) { + uint32_t tag = (r << 16) | (trace->wid << 4) | (int)RegType::Float; + auto owner = owners_.at(tag); + out.push_back({RegType::Float, r, owner->exe_type, owner->sfu_type, owner->uuid}); } } - { - uint32_t r = 0; - auto used_vregs = state->used_vregs & in_use_vregs_.at(state->wid); - while (used_vregs.any()) { - if (used_vregs.test(0)) { - uint32_t tag = (r << 16) | (state->wid << 4) | (int)RegType::Vector; - out.push_back({RegType::Vector, r, owners_.at(tag)}); - } - used_vregs >>= 1; - ++r; + + for (uint32_t r = 0; r < MAX_NUM_REGS; ++r) { + if (used_vregs.test(r)) { + uint32_t tag = (r << 16) | (trace->wid << 4) | (int)RegType::Vector; + auto owner = owners_.at(tag); + out.push_back({RegType::Vector, r, owner->exe_type, owner->sfu_type, owner->uuid}); } } + return out; } - void reserve(pipeline_trace_t* state) { - assert(state->wb); - switch (state->rdest_type) { + void reserve(pipeline_trace_t* trace) { + assert(trace->wb); + switch (trace->rdest_type) { case RegType::Integer: - in_use_iregs_.at(state->wid).set(state->rdest); + in_use_iregs_.at(trace->wid).set(trace->rdest); break; case RegType::Float: - in_use_fregs_.at(state->wid).set(state->rdest); + in_use_fregs_.at(trace->wid).set(trace->rdest); break; case RegType::Vector: - in_use_vregs_.at(state->wid).set(state->rdest); - break; - default: + in_use_vregs_.at(trace->wid).set(trace->rdest); break; + default: assert(false); } - uint32_t tag = (state->rdest << 16) | (state->wid << 4) | (int)state->rdest_type; + uint32_t tag = (trace->rdest << 16) | (trace->wid << 4) | (int)trace->rdest_type; assert(owners_.count(tag) == 0); - owners_[tag] = state->uuid; + owners_[tag] = trace; + assert((int)trace->exe_type < 5); } - void release(pipeline_trace_t* state) { - assert(state->wb); - switch (state->rdest_type) { + void release(pipeline_trace_t* trace) { + assert(trace->wb); + switch (trace->rdest_type) { case RegType::Integer: - in_use_iregs_.at(state->wid).reset(state->rdest); + in_use_iregs_.at(trace->wid).reset(trace->rdest); break; case RegType::Float: - in_use_fregs_.at(state->wid).reset(state->rdest); + in_use_fregs_.at(trace->wid).reset(trace->rdest); break; case RegType::Vector: - in_use_vregs_.at(state->wid).reset(state->rdest); - break; - default: + in_use_vregs_.at(trace->wid).reset(trace->rdest); break; + default: assert(false); } - uint32_t tag = (state->rdest << 16) | (state->wid << 4) | (int)state->rdest_type; + uint32_t tag = (trace->rdest << 16) | (trace->wid << 4) | (int)trace->rdest_type; owners_.erase(tag); } @@ -135,7 +129,7 @@ private: std::vector in_use_iregs_; std::vector in_use_fregs_; std::vector in_use_vregs_; - std::unordered_map owners_; + std::unordered_map owners_; }; } \ No newline at end of file diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp new file mode 100644 index 00000000..dd9f9697 --- /dev/null +++ b/sim/simx/socket.cpp @@ -0,0 +1,149 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "socket.h" +#include "cluster.h" + +using namespace vortex; + +Socket::Socket(const SimContext& ctx, + uint32_t socket_id, + Cluster* cluster, + const Arch &arch, + const DCRS &dcrs) + : SimObject(ctx, "socket") + , icache_mem_req_port(this) + , icache_mem_rsp_port(this) + , dcache_mem_req_port(this) + , dcache_mem_rsp_port(this) + , socket_id_(socket_id) + , cluster_(cluster) + , cores_(arch.socket_size()) +{ + auto cores_per_socket = cores_.size(); + + char sname[100]; + snprintf(sname, 100, "socket%d-icaches", socket_id); + icaches_ = CacheCluster::Create(sname, cores_per_socket, NUM_ICACHES, 1, CacheSim::Config{ + !ICACHE_ENABLED, + log2ceil(ICACHE_SIZE), // C + log2ceil(L1_LINE_SIZE), // L + log2ceil(sizeof(uint32_t)), // W + log2ceil(ICACHE_NUM_WAYS),// A + 1, // B + XLEN, // address bits + 1, // number of ports + 1, // number of inputs + true, // write-through + false, // write response + (uint8_t)arch.num_warps(), // mshr + 2, // pipeline latency + }); + + icaches_->MemReqPort.bind(&icache_mem_req_port); + icache_mem_rsp_port.bind(&icaches_->MemRspPort); + + snprintf(sname, 100, "socket%d-dcaches", socket_id); + dcaches_ = CacheCluster::Create(sname, cores_per_socket, NUM_DCACHES, NUM_LSU_LANES, CacheSim::Config{ + !DCACHE_ENABLED, + log2ceil(DCACHE_SIZE), // C + log2ceil(L1_LINE_SIZE), // L + log2ceil(sizeof(Word)), // W + log2ceil(DCACHE_NUM_WAYS),// A + log2ceil(DCACHE_NUM_BANKS), // B + XLEN, // address bits + 1, // number of ports + DCACHE_NUM_BANKS, // number of inputs + true, // write-through + false, // write response + DCACHE_MSHR_SIZE, // mshr + 2, // pipeline latency + }); + + dcaches_->MemReqPort.bind(&dcache_mem_req_port); + dcache_mem_rsp_port.bind(&dcaches_->MemRspPort); + + // create cores + + for (uint32_t i = 0; i < cores_per_socket; ++i) { + uint32_t core_id = socket_id * cores_per_socket + i; + cores_.at(i) = Core::Create(core_id, + this, + arch, + dcrs); + + cores_.at(i)->icache_req_ports.at(0).bind(&icaches_->CoreReqPorts.at(i).at(0)); + icaches_->CoreRspPorts.at(i).at(0).bind(&cores_.at(i)->icache_rsp_ports.at(0)); + + for (uint32_t j = 0; j < NUM_LSU_LANES; ++j) { + cores_.at(i)->dcache_req_ports.at(j).bind(&dcaches_->CoreReqPorts.at(i).at(j)); + dcaches_->CoreRspPorts.at(i).at(j).bind(&cores_.at(i)->dcache_rsp_ports.at(j)); + } + } +} + +Socket::~Socket() { + //-- +} + +void Socket::reset() { + //-- +} + +void Socket::tick() { + //-- +} + +void Socket::attach_ram(RAM* ram) { + for (auto core : cores_) { + core->attach_ram(ram); + } +} + +bool Socket::running() const { + for (auto& core : cores_) { + if (core->running()) + return true; + } + return false; +} + +bool Socket::check_exit(Word* exitcode, bool riscv_test) const { + bool done = true; + Word exitcode_ = 0; + for (auto& core : cores_) { + Word ec; + if (core->check_exit(&ec, riscv_test)) { + exitcode_ |= ec; + } else { + done = false; + } + } + *exitcode = exitcode_; + return done; +} + +void Socket::barrier(uint32_t bar_id, uint32_t count, uint32_t core_id) { + cluster_->barrier(bar_id, count, socket_id_ * cores_.size() + core_id); +} + +void Socket::resume(uint32_t core_index) { + cores_.at(core_index)->resume(); +} + +Socket::PerfStats Socket::perf_stats() const { + PerfStats perf_stats; + perf_stats.icache = icaches_->perf_stats(); + perf_stats.dcache = dcaches_->perf_stats(); + return perf_stats; +} \ No newline at end of file diff --git a/sim/simx/socket.h b/sim/simx/socket.h new file mode 100644 index 00000000..5105f99e --- /dev/null +++ b/sim/simx/socket.h @@ -0,0 +1,81 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "dcrs.h" +#include "arch.h" +#include "cache_cluster.h" +#include "shared_mem.h" +#include "core.h" +#include "constants.h" + +namespace vortex { + +class Cluster; + +class Socket : public SimObject { +public: + struct PerfStats { + CacheSim::PerfStats icache; + CacheSim::PerfStats dcache; + }; + + SimPort icache_mem_req_port; + SimPort icache_mem_rsp_port; + + SimPort dcache_mem_req_port; + SimPort dcache_mem_rsp_port; + + Socket(const SimContext& ctx, + uint32_t socket_id, + Cluster* cluster, + const Arch &arch, + const DCRS &dcrs); + + ~Socket(); + + uint32_t id() const { + return socket_id_; + } + + Cluster* cluster() const { + return cluster_; + } + + void reset(); + + void tick(); + + void attach_ram(RAM* ram); + + bool running() const; + + bool check_exit(Word* exitcode, bool riscv_test) const; + + void barrier(uint32_t bar_id, uint32_t count, uint32_t core_id); + + void resume(uint32_t core_id); + + PerfStats perf_stats() const; + +private: + uint32_t socket_id_; + Cluster* cluster_; + std::vector cores_; + CacheCluster::Ptr icaches_; + CacheCluster::Ptr dcaches_; +}; + +} // namespace vortex \ No newline at end of file diff --git a/sim/simx/types.h b/sim/simx/types.h index 88b3ce0e..d3fcfa1a 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -70,6 +70,7 @@ inline std::ostream &operator<<(std::ostream &os, const RegType& type) { case RegType::Integer: os << "x"; break; case RegType::Float: os << "f"; break; case RegType::Vector: os << "v"; break; + default: assert(false); } return os; } @@ -81,7 +82,7 @@ enum class ExeType { LSU, FPU, SFU, - MAX, + ExeTypeCount }; inline std::ostream &operator<<(std::ostream &os, const ExeType& type) { @@ -90,7 +91,7 @@ inline std::ostream &operator<<(std::ostream &os, const ExeType& type) { case ExeType::LSU: os << "LSU"; break; case ExeType::FPU: os << "FPU"; break; case ExeType::SFU: os << "SFU"; break; - case ExeType::MAX: break; + default: assert(false); } return os; } @@ -112,6 +113,7 @@ inline std::ostream &operator<<(std::ostream &os, const AluType& type) { case AluType::SYSCALL: os << "SYSCALL"; break; case AluType::IMUL: os << "IMUL"; break; case AluType::IDIV: os << "IDIV"; break; + default: assert(false); } return os; } @@ -129,6 +131,7 @@ inline std::ostream &operator<<(std::ostream &os, const LsuType& type) { case LsuType::LOAD: os << "LOAD"; break; case LsuType::STORE: os << "STORE"; break; case LsuType::FENCE: os << "FENCE"; break; + default: assert(false); } return os; } @@ -138,7 +141,7 @@ inline std::ostream &operator<<(std::ostream &os, const LsuType& type) { enum class AddrType { Global, Shared, - IO, + IO }; inline std::ostream &operator<<(std::ostream &os, const AddrType& type) { @@ -146,6 +149,7 @@ inline std::ostream &operator<<(std::ostream &os, const AddrType& type) { case AddrType::Global: os << "Global"; break; case AddrType::Shared: os << "Shared"; break; case AddrType::IO: os << "IO"; break; + default: assert(false); } return os; } @@ -164,7 +168,7 @@ enum class FpuType { FMA, FDIV, FSQRT, - FCVT, + FCVT }; inline std::ostream &operator<<(std::ostream &os, const FpuType& type) { @@ -174,6 +178,7 @@ inline std::ostream &operator<<(std::ostream &os, const FpuType& type) { case FpuType::FDIV: os << "FDIV"; break; case FpuType::FSQRT: os << "FSQRT"; break; case FpuType::FCVT: os << "FCVT"; break; + default: assert(false); } return os; } @@ -190,7 +195,7 @@ enum class SfuType { CSRRW, CSRRS, CSRRC, - CMOV + CMOV }; inline std::ostream &operator<<(std::ostream &os, const SfuType& type) { @@ -205,6 +210,7 @@ inline std::ostream &operator<<(std::ostream &os, const SfuType& type) { case SfuType::CSRRS: os << "CSRRS"; break; case SfuType::CSRRC: os << "CSRRC"; break; case SfuType::CMOV: os << "CMOV"; break; + default: assert(false); } return os; } @@ -220,6 +226,7 @@ inline std::ostream &operator<<(std::ostream &os, const ArbiterType& type) { switch (type) { case ArbiterType::Priority: os << "Priority"; break; case ArbiterType::RoundRobin: os << "RoundRobin"; break; + default: assert(false); } return os; } @@ -351,6 +358,92 @@ private: /////////////////////////////////////////////////////////////////////////////// +template +class Mux : public SimObject> { +public: + std::vector> Inputs; + std::vector> Outputs; + + Mux( + const SimContext& ctx, + const char* name, + ArbiterType type, + uint32_t num_inputs, + uint32_t num_outputs = 1, + uint32_t delay = 1 + ) : SimObject>(ctx, name) + , Inputs(num_inputs, this) + , Outputs(num_outputs, this) + , type_(type) + , delay_(delay) + , cursors_(num_outputs, 0) + , num_reqs_(num_inputs / num_outputs) + { + assert(delay != 0); + assert(num_inputs <= 32); + assert(num_outputs <= 32); + assert(num_inputs >= num_outputs); + + // bypass mode + if (num_inputs == num_outputs) { + for (uint32_t i = 0; i < num_inputs; ++i) { + Inputs.at(i).bind(&Outputs.at(i)); + } + } + } + + void reset() { + for (auto& cursor : cursors_) { + cursor = 0; + } + } + + void tick() { + uint32_t I = Inputs.size(); + uint32_t O = Outputs.size(); + uint32_t R = num_reqs_; + + // skip bypass mode + if (I == O) + return; + + // process inputs + for (uint32_t o = 0; o < O; ++o) { + for (uint32_t r = 0; r < R; ++r) { + uint32_t i = (cursors_.at(o) + r) & (R-1); + uint32_t j = o * R + i; + if (j >= I) + continue; + + auto& req_in = Inputs.at(j); + if (!req_in.empty()) { + auto& req = req_in.front(); + DT(4, this->name() << "-" << req); + Outputs.at(o).send(req, delay_); + req_in.pop(); + this->update_cursor(o, i); + break; + } + } + } + } + +private: + + void update_cursor(uint32_t index, uint32_t grant) { + if (type_ == ArbiterType::RoundRobin) { + cursors_.at(index) = grant + 1; + } + } + + ArbiterType type_; + uint32_t delay_; + std::vector cursors_; + uint32_t num_reqs_; +}; + +/////////////////////////////////////////////////////////////////////////////// + template class Switch : public SimObject> { public: @@ -364,13 +457,13 @@ public: const SimContext& ctx, const char* name, ArbiterType type, - uint32_t num_inputs = 1, + uint32_t num_inputs, uint32_t num_outputs = 1, uint32_t delay = 1 ) : SimObject>(ctx, name) - , ReqIn(num_inputs, this) - , RspIn(num_inputs, this) + , ReqIn(num_inputs, this) + , RspIn(num_inputs, this) , ReqOut(num_outputs, this) , RspOut(num_outputs, this) , type_(type) @@ -383,8 +476,8 @@ public: assert(num_outputs <= 32); assert(num_inputs >= num_outputs); + // bypass mode if (num_inputs == num_outputs) { - // bypass mode for (uint32_t i = 0; i < num_inputs; ++i) { ReqIn.at(i).bind(&ReqOut.at(i)); RspOut.at(i).bind(&RspIn.at(i)); @@ -462,14 +555,14 @@ private: class SMemDemux : public SimObject { public: - SimPort ReqIn; - SimPort RspIn; + SimPort ReqIn; + SimPort RspIn; - SimPort ReqSm; - SimPort RspSm; + SimPort ReqSM; + SimPort RspSM; - SimPort ReqDc; - SimPort RspDc; + SimPort ReqDC; + SimPort RspDC; SMemDemux( const SimContext& ctx, @@ -478,45 +571,49 @@ public: ) : SimObject(ctx, name) , ReqIn(this) , RspIn(this) - , ReqSm(this) - , RspSm(this) - , ReqDc(this) - , RspDc(this) + , ReqSM(this) + , RspSM(this) + , ReqDC(this) + , RspDC(this) , delay_(delay) {} void reset() {} - void tick() { + void tick() { + // process incoming reponses + if (!RspSM.empty()) { + auto& rsp = RspSM.front(); + DT(4, this->name() << "-" << rsp); + RspIn.send(rsp, 1); + RspSM.pop(); + } + if (!RspDC.empty()) { + auto& rsp = RspDC.front(); + DT(4, this->name() << "-" << rsp); + RspIn.send(rsp, 1); + RspDC + .pop(); + } // process incomming requests if (!ReqIn.empty()) { auto& req = ReqIn.front(); DT(4, this->name() << "-" << req); if (req.type == AddrType::Shared) { - ReqSm.send(req, delay_); + ReqSM.send(req, delay_); } else { - ReqDc.send(req, delay_); + ReqDC.send(req, delay_); } ReqIn.pop(); } - - // process incoming reponses - if (!RspSm.empty()) { - auto& rsp = RspSm.front(); - DT(4, this->name() << "-" << rsp); - RspIn.send(rsp, 1); - RspSm.pop(); - } - if (!RspDc.empty()) { - auto& rsp = RspDc.front(); - DT(4, this->name() << "-" << rsp); - RspIn.send(rsp, 1); - RspDc.pop(); - } } private: uint32_t delay_; }; -} \ No newline at end of file +/////////////////////////////////////////////////////////////////////////////// + +using MemSwitch = Switch; + +} diff --git a/tests/kernel/common.mk b/tests/kernel/common.mk new file mode 100644 index 00000000..7bf4b520 --- /dev/null +++ b/tests/kernel/common.mk @@ -0,0 +1,49 @@ +XLEN ?= 32 +TOOLDIR ?= /opt + +ifeq ($(XLEN),64) +RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv64-gnu-toolchain +CFLAGS += -march=rv64imafd -mabi=lp64d +else +RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv-gnu-toolchain +CFLAGS += -march=rv32imaf -mabi=ilp32f +endif + +RISCV_PREFIX ?= riscv$(XLEN)-unknown-elf + +VORTEX_KN_PATH ?= $(realpath ../../../kernel) + +CC = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-gcc +AR = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-gcc-ar +DP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objdump +CP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objcopy + +SIM_DIR = ../../../sim + +CFLAGS += -O3 -mcmodel=medany -fno-exceptions -nostartfiles -fdata-sections -ffunction-sections +CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw + +LDFLAGS += -lm -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=0x80000000 $(VORTEX_KN_PATH)/libvortexrt.a + +all: $(PROJECT).elf $(PROJECT).bin $(PROJECT).dump + +$(PROJECT).dump: $(PROJECT).elf + $(DP) -D $(PROJECT).elf > $(PROJECT).dump + +$(PROJECT).bin: $(PROJECT).elf + $(CP) -O binary $(PROJECT).elf $(PROJECT).bin + +$(PROJECT).elf: $(SRCS) + $(CC) $(CFLAGS) $(SRCS) $(LDFLAGS) -o $(PROJECT).elf + +run-rtlsim: $(PROJECT).bin + $(SIM_DIR)/rtlsim/rtlsim $(PROJECT).bin + +run-simx: $(PROJECT).bin + $(SIM_DIR)/simx/simx $(PROJECT).bin + +.depend: $(SRCS) + $(CC) $(CFLAGS) -MM $^ > .depend; + +clean: + rm -rf *.elf *.bin *.dump .depend diff --git a/tests/kernel/conform/Makefile b/tests/kernel/conform/Makefile index e9897125..ee96978f 100644 --- a/tests/kernel/conform/Makefile +++ b/tests/kernel/conform/Makefile @@ -1,52 +1,5 @@ -XLEN ?= 32 - -ifeq ($(XLEN),64) -RISCV_TOOLCHAIN_PATH ?= /opt/riscv64-gnu-toolchain -CFLAGS += -march=rv64imafd -mabi=lp64d -else -RISCV_TOOLCHAIN_PATH ?= /opt/riscv-gnu-toolchain -CFLAGS += -march=rv32imaf -mabi=ilp32f -endif - -RISCV_PREFIX ?= riscv$(XLEN)-unknown-elf - -VORTEX_KN_PATH ?= $(realpath ../../../kernel) - -CC = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-gcc -AR = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-gcc-ar -DP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objdump -CP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objcopy - -SIM_DIR = ../../../sim - -CFLAGS += -O3 -mcmodel=medany -fno-exceptions -nostartfiles -fdata-sections -ffunction-sections -CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw - -LDFLAGS += -lm -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=0x80000000 $(VORTEX_KN_PATH)/libvortexrt.a - PROJECT = conform SRCS = main.cpp tests.cpp -all: $(PROJECT).elf $(PROJECT).bin $(PROJECT).dump - -$(PROJECT).dump: $(PROJECT).elf - $(DP) -D $(PROJECT).elf > $(PROJECT).dump - -$(PROJECT).bin: $(PROJECT).elf - $(CP) -O binary $(PROJECT).elf $(PROJECT).bin - -$(PROJECT).elf: $(SRCS) - $(CC) $(CFLAGS) $(SRCS) $(LDFLAGS) -o $(PROJECT).elf - -run-rtlsim: $(PROJECT).bin - $(SIM_DIR)/rtlsim/rtlsim $(PROJECT).bin - -run-simx: $(PROJECT).bin - $(SIM_DIR)/simx/simx $(PROJECT).bin - -.depend: $(SRCS) - $(CC) $(CFLAGS) -MM $^ > .depend; - -clean: - rm -rf *.elf *.bin *.dump .depend +include ../common.mk diff --git a/tests/kernel/fibonacci/Makefile b/tests/kernel/fibonacci/Makefile index 1338b4ab..d4486c74 100644 --- a/tests/kernel/fibonacci/Makefile +++ b/tests/kernel/fibonacci/Makefile @@ -1,52 +1,5 @@ -XLEN ?= 32 - -ifeq ($(XLEN),64) -RISCV_TOOLCHAIN_PATH ?= /opt/riscv64-gnu-toolchain -CFLAGS += -march=rv64imafd -mabi=lp64d -else -RISCV_TOOLCHAIN_PATH ?= /opt/riscv-gnu-toolchain -CFLAGS += -march=rv32imaf -mabi=ilp32f -endif - -RISCV_PREFIX ?= riscv$(XLEN)-unknown-elf - -VORTEX_KN_PATH ?= $(realpath ../../../kernel) - -CC = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-gcc -AR = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-gcc-ar -DP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objdump -CP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objcopy - -SIM_DIR = ../../../sim - -CFLAGS += -O3 -mcmodel=medany -fno-exceptions -nostartfiles -fdata-sections -ffunction-sections -CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw - -LDFLAGS += -lm -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=0x80000000 $(VORTEX_KN_PATH)/libvortexrt.a - PROJECT = fibonacci SRCS = main.cpp -all: $(PROJECT).elf $(PROJECT).bin $(PROJECT).dump - -$(PROJECT).dump: $(PROJECT).elf - $(DP) -D $(PROJECT).elf > $(PROJECT).dump - -$(PROJECT).bin: $(PROJECT).elf - $(CP) -O binary $(PROJECT).elf $(PROJECT).bin - -$(PROJECT).elf: $(SRCS) - $(CC) $(CFLAGS) $(SRCS) $(LDFLAGS) -o $(PROJECT).elf - -run-rtlsim: $(PROJECT).bin - $(SIM_DIR)/rtlsim/rtlsim $(PROJECT).bin - -run-simx: $(PROJECT).bin - $(SIM_DIR)/simx/simx $(PROJECT).bin - -.depend: $(SRCS) - $(CC) $(CFLAGS) -MM $^ > .depend; - -clean: - rm -rf *.elf *.bin *.dump .depend +include ../common.mk diff --git a/tests/kernel/hello/Makefile b/tests/kernel/hello/Makefile index 42d95256..4cff6cbd 100644 --- a/tests/kernel/hello/Makefile +++ b/tests/kernel/hello/Makefile @@ -1,52 +1,5 @@ -XLEN ?= 32 - -ifeq ($(XLEN),64) -RISCV_TOOLCHAIN_PATH ?= /opt/riscv64-gnu-toolchain -CFLAGS += -march=rv64imafd -mabi=lp64d -else -RISCV_TOOLCHAIN_PATH ?= /opt/riscv-gnu-toolchain -CFLAGS += -march=rv32imaf -mabi=ilp32f -endif - -RISCV_PREFIX ?= riscv$(XLEN)-unknown-elf - -VORTEX_KN_PATH ?= $(realpath ../../../kernel) - -CC = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-gcc -AR = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-gcc-ar -DP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objdump -CP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objcopy - -SIM_DIR = ../../../sim - -CFLAGS += -O3 -v -mcmodel=medany -fno-exceptions -nostartfiles -fdata-sections -ffunction-sections -CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw - -LDFLAGS += -lm -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=0x80000000 $(VORTEX_KN_PATH)/libvortexrt.a - PROJECT = hello SRCS = main.cpp -all: $(PROJECT).elf $(PROJECT).bin $(PROJECT).dump - -$(PROJECT).dump: $(PROJECT).elf - $(DP) -D $(PROJECT).elf > $(PROJECT).dump - -$(PROJECT).bin: $(PROJECT).elf - $(CP) -O binary $(PROJECT).elf $(PROJECT).bin - -$(PROJECT).elf: $(SRCS) - $(CC) $(CFLAGS) $(SRCS) $(LDFLAGS) -o $(PROJECT).elf - -run-rtlsim: $(PROJECT).bin - $(SIM_DIR)/rtlsim/rtlsim $(PROJECT).bin - -run-simx: $(PROJECT).bin - $(SIM_DIR)/simx/simx $(PROJECT).bin - -.depend: $(SRCS) - $(CC) $(CFLAGS) -MM $^ > .depend; - -clean: - rm -rf *.elf *.bin *.dump .depend +include ../common.mk diff --git a/tests/opencl/Makefile b/tests/opencl/Makefile index 2cee5c5d..5635643e 100644 --- a/tests/opencl/Makefile +++ b/tests/opencl/Makefile @@ -9,15 +9,12 @@ all: $(MAKE) -C dotproduct $(MAKE) -C kmeans $(MAKE) -C spmv - $(MAKE) -C transpose - $(MAKE) -C cutcp - $(MAKE) -C vectorhypot $(MAKE) -C stencil - $(MAKE) -C mri-q $(MAKE) -C lbm $(MAKE) -C oclprintf $(MAKE) -C blackscholes - $(MAKE) -C matmul + $(MAKE) -C transpose + $(MAKE) -C convolution run-simx: $(MAKE) -C vecadd run-simx @@ -30,15 +27,12 @@ run-simx: $(MAKE) -C dotproduct run-simx $(MAKE) -C kmeans run-simx $(MAKE) -C spmv run-simx - $(MAKE) -C cutcp run-simx $(MAKE) -C stencil run-simx $(MAKE) -C lbm run-simx $(MAKE) -C oclprintf run-simx $(MAKE) -C blackscholes run-simx - $(MAKE) -C matmul run-simx $(MAKE) -C transpose run-simx -# $(MAKE) -C vectorhypot run-simx -# $(MAKE) -C mri-q run-simx + $(MAKE) -C convolution run-simx run-rtlsim: $(MAKE) -C vecadd run-rtlsim @@ -52,14 +46,11 @@ run-rtlsim: $(MAKE) -C kmeans run-rtlsim $(MAKE) -C spmv run-rtlsim $(MAKE) -C transpose run-rtlsim - $(MAKE) -C cutcp run-rtlsim $(MAKE) -C stencil run-rtlsim $(MAKE) -C lbm run-rtlsim $(MAKE) -C oclprintf run-rtlsim $(MAKE) -C blackscholes run-rtlsim - $(MAKE) -C matmul run-rtlsim -# $(MAKE) -C vectorhypot run-rtlsim -# $(MAKE) -C mri-q run-rtlsim + $(MAKE) -C convolution run-rtlsim run-opae: $(MAKE) -C vecadd run-opae @@ -73,14 +64,11 @@ run-opae: $(MAKE) -C kmeans run-opae $(MAKE) -C spmv run-opae $(MAKE) -C transpose run-opae - $(MAKE) -C cutcp run-opae $(MAKE) -C stencil run-opae $(MAKE) -C lbm run-opae $(MAKE) -C oclprintf run-opae $(MAKE) -C blackscholes run-opae - $(MAKE) -C matmul run-opae -# $(MAKE) -C vectorhypot run-opae -# $(MAKE) -C mri-q run-opae + $(MAKE) -C convolution run-opae clean: $(MAKE) -C vecadd clean @@ -94,14 +82,11 @@ clean: $(MAKE) -C kmeans clean $(MAKE) -C spmv clean $(MAKE) -C transpose clean - $(MAKE) -C cutcp clean - $(MAKE) -C vectorhypot clean $(MAKE) -C stencil clean - $(MAKE) -C mri-q clean $(MAKE) -C lbm clean $(MAKE) -C oclprintf clean $(MAKE) -C blackscholes clean - $(MAKE) -C matmul clean + $(MAKE) -C convolution clean clean-all: $(MAKE) -C vecadd clean-all @@ -109,18 +94,14 @@ clean-all: $(MAKE) -C psort clean-all $(MAKE) -C saxpy clean-all $(MAKE) -C sfilter clean-all - $(MAKE) -C sfilter clean-all $(MAKE) -C nearn clean-all $(MAKE) -C guassian clean-all $(MAKE) -C dotproduct clean-all $(MAKE) -C kmeans clean-all $(MAKE) -C spmv clean-all $(MAKE) -C transpose clean-all - $(MAKE) -C cutcp clean-all - $(MAKE) -C vectorhypot clean-all $(MAKE) -C stencil clean-all - $(MAKE) -C mri-q clean-all $(MAKE) -C lbm clean-all $(MAKE) -C oclprintf clean-all $(MAKE) -C blackscholes clean-all - $(MAKE) -C matmul clean-all + $(MAKE) -C convolution clean-all \ No newline at end of file diff --git a/tests/opencl/common.mk b/tests/opencl/common.mk index fc55ee0a..cfb436a6 100644 --- a/tests/opencl/common.mk +++ b/tests/opencl/common.mk @@ -1,4 +1,5 @@ XLEN ?= 32 +TOOLDIR ?= /opt TARGET ?= opaesim @@ -6,12 +7,12 @@ XRT_SYN_DIR ?= ../../../hw/syn/xilinx/xrt XRT_DEVICE_INDEX ?= 0 ifeq ($(XLEN),64) -RISCV_TOOLCHAIN_PATH ?= /opt/riscv64-gnu-toolchain +RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv64-gnu-toolchain VX_CFLAGS += -march=rv64imafd -mabi=lp64d K_CFLAGS += -march=rv64imafd -mabi=ilp64d STARTUP_ADDR ?= 0x180000000 else -RISCV_TOOLCHAIN_PATH ?= /opt/riscv-gnu-toolchain +RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv-gnu-toolchain VX_CFLAGS += -march=rv32imaf -mabi=ilp32f K_CFLAGS += -march=rv32imaf -mabi=ilp32f STARTUP_ADDR ?= 0x80000000 @@ -20,16 +21,16 @@ endif RISCV_PREFIX ?= riscv$(XLEN)-unknown-elf RISCV_SYSROOT ?= $(RISCV_TOOLCHAIN_PATH)/$(RISCV_PREFIX) -POCL_CC_PATH ?= /opt/pocl/compiler -POCL_RT_PATH ?= /opt/pocl/runtime +POCL_CC_PATH ?= $(TOOLDIR)/pocl/compiler +POCL_RT_PATH ?= $(TOOLDIR)/pocl/runtime VORTEX_RT_PATH ?= $(realpath ../../../runtime) VORTEX_KN_PATH ?= $(realpath ../../../kernel) FPGA_BIN_DIR ?= $(VORTEX_RT_PATH)/opae -LLVM_VORTEX ?= /opt/llvm-vortex -LLVM_POCL ?= /opt/llvm-vortex +LLVM_VORTEX ?= $(TOOLDIR)/llvm-vortex +LLVM_POCL ?= $(TOOLDIR)/llvm-vortex K_CFLAGS += -v -O3 --sysroot=$(RISCV_SYSROOT) --gcc-toolchain=$(RISCV_TOOLCHAIN_PATH) -Xclang -target-feature -Xclang +vortex K_CFLAGS += -fno-rtti -fno-exceptions -nostartfiles -fdata-sections -ffunction-sections @@ -40,13 +41,12 @@ CXXFLAGS += -std=c++11 -Wall -Wextra -Wfatal-errors CXXFLAGS += -Wno-deprecated-declarations -Wno-unused-parameter -Wno-narrowing CXXFLAGS += -pthread CXXFLAGS += -I$(POCL_RT_PATH)/include -LDFLAGS += -L$(POCL_RT_PATH)/lib -L$(VORTEX_RT_PATH)/stub -lvortex ifdef HOSTGPU CXXFLAGS += -DHOSTGPU LDFLAGS += -lOpenCL else - LDFLAGS += $(POCL_RT_PATH)/lib/libOpenCL.so + LDFLAGS += -L$(VORTEX_RT_PATH)/stub -lvortex $(POCL_RT_PATH)/lib/libOpenCL.so endif # Debugigng @@ -73,7 +73,7 @@ OBJS := $(addsuffix .o, $(notdir $(SRCS))) all: $(PROJECT) kernel.pocl kernel.pocl: kernel.cl - LLVM_PREFIX=$(LLVM_VORTEX) POCL_DEBUG=all LD_LIBRARY_PATH=$(LLVM_POCL)/lib:$(POCL_CC_PATH)/lib:$(LLVM_VORTEX)/lib POCL_VORTEX_CFLAGS="$(K_CFLAGS)" POCL_VORTEX_LDFLAGS="$(K_LDFLAGS)" $(POCL_CC_PATH)/bin/poclcc -o kernel.pocl kernel.cl + LD_LIBRARY_PATH=$(LLVM_POCL)/lib:$(POCL_CC_PATH)/lib:$(LLVM_VORTEX)/lib:$(LD_LIBRARY_PATH) LLVM_PREFIX=$(LLVM_VORTEX) POCL_DEBUG=all POCL_VORTEX_CFLAGS="$(K_CFLAGS)" POCL_VORTEX_LDFLAGS="$(K_LDFLAGS)" $(POCL_CC_PATH)/bin/poclcc -o kernel.pocl kernel.cl %.cc.o: %.cc $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -87,6 +87,9 @@ kernel.pocl: kernel.cl $(PROJECT): $(OBJS) $(CXX) $(CXXFLAGS) $^ $(LDFLAGS) -o $@ +run-hostgpu: $(PROJECT) kernel.pocl + ./$(PROJECT) $(OPTS) + run-simx: $(PROJECT) kernel.pocl LD_LIBRARY_PATH=$(POCL_RT_PATH)/lib:$(VORTEX_RT_PATH)/simx:$(LD_LIBRARY_PATH) ./$(PROJECT) $(OPTS) diff --git a/tests/opencl/matmul/Makefile b/tests/opencl/convolution/Makefile similarity index 52% rename from tests/opencl/matmul/Makefile rename to tests/opencl/convolution/Makefile index 0d1d136a..42a577d2 100644 --- a/tests/opencl/matmul/Makefile +++ b/tests/opencl/convolution/Makefile @@ -1,7 +1,7 @@ -PROJECT = matmul +PROJECT = convolution SRCS = main.cc -OPTS ?= -n16 +OPTS ?= -n32 include ../common.mk diff --git a/tests/opencl/convolution/kernel.cl b/tests/opencl/convolution/kernel.cl new file mode 100644 index 00000000..2ef31040 --- /dev/null +++ b/tests/opencl/convolution/kernel.cl @@ -0,0 +1,32 @@ +__kernel void conv3x3(__global float* output, + __global float* input, + __global float* weights, + const int width, + const int height) +{ + int x = get_global_id(0); + int y = get_global_id(1); + + // Adjust for padded borders + int paddedWidth = width + 2; + int paddedX = x + 1; + int paddedY = y + 1; + + // Compute the convolution sum + float sum = 0.0f; + + sum += input[(paddedY - 1) * paddedWidth + (paddedX - 1)] * weights[0]; // Top-left + sum += input[(paddedY - 1) * paddedWidth + paddedX] * weights[1]; // Top-center + sum += input[(paddedY - 1) * paddedWidth + (paddedX + 1)] * weights[2]; // Top-right + + sum += input[paddedY * paddedWidth + (paddedX - 1)] * weights[3]; // Middle-left + sum += input[paddedY * paddedWidth + paddedX] * weights[4]; // Center + sum += input[paddedY * paddedWidth + (paddedX + 1)] * weights[5]; // Middle-right + + sum += input[(paddedY + 1) * paddedWidth + (paddedX - 1)] * weights[6]; // Bottom-left + sum += input[(paddedY + 1) * paddedWidth + paddedX] * weights[7]; // Bottom-center + sum += input[(paddedY + 1) * paddedWidth + (paddedX + 1)] * weights[8]; // Bottom-right + + // Store the result in the output array + output[y * width + x] = sum; +} \ No newline at end of file diff --git a/tests/opencl/matmul/main.cc b/tests/opencl/convolution/main.cc similarity index 65% rename from tests/opencl/matmul/main.cc rename to tests/opencl/convolution/main.cc index f7657a60..5c62b56e 100644 --- a/tests/opencl/matmul/main.cc +++ b/tests/opencl/convolution/main.cc @@ -8,9 +8,9 @@ #include #include -#define LOCAL_SIZE 16 +#define FLOAT_ULP 6 -#define KERNEL_NAME "matmul" +#define KERNEL_NAME "conv3x3" #define CL_CHECK(_expr) \ do { \ @@ -77,24 +77,34 @@ static int write_operand_file(const char* filename, void* data, size_t size) { return 0; } -static bool compare_equal(float a, float b, int ulp = 21) { - union fi_t { int i; float f; }; +static bool compare_equal(float a, float b) { + union fi_t { float f; int32_t i; }; fi_t fa, fb; fa.f = a; fb.f = b; - return std::abs(fa.i - fb.i) <= ulp; + auto d = std::abs(fa.i - fb.i); + return d <= FLOAT_ULP; } -static void matrix_multiply_cpu(float *A, float *B, float *C, int N) { - for (int i = 0; i < N; i++) { - for (int j = 0; j < N; j++) { - float sum = 0.0f; - for (int k = 0; k < N; k++) { - sum += A[i * N + k] * B[k * N + j]; - } - C[i * N + j] = sum; +static void convolution_cpu(float *O, float *I, float *W, int32_t width, int32_t height) { + int paddedWidth = width + 2; + for (int32_t y = 0; y < height; ++y) { + for (int32_t x = 0; x < width; ++x) { + int paddedY = y + 1; + int paddedX = x + 1; + float sum = 0.0f; + for (int32_t ky = -1; ky <= 1; ++ky) { + for (int32_t kx = -1; kx <= 1; ++kx) { + int32_t iy = paddedY + ky; + int32_t ix = paddedX + kx; + float value = I[iy * paddedWidth + ix]; + float weight = W[(ky + 1) * 3 + (kx + 1)]; + sum += value * weight; } + } + O[y * width + x] = sum; } + } } cl_device_id device_id = NULL; @@ -102,24 +112,24 @@ cl_context context = NULL; cl_command_queue commandQueue = NULL; cl_program program = NULL; cl_kernel kernel = NULL; -cl_mem a_memobj = NULL; -cl_mem b_memobj = NULL; -cl_mem c_memobj = NULL; -uint8_t *kernel_bin = NULL; +cl_mem i_memobj = NULL; +cl_mem w_memobj = NULL; +cl_mem o_memobj = NULL; +uint8_t* kernel_bin = NULL; static void cleanup() { if (commandQueue) clReleaseCommandQueue(commandQueue); if (kernel) clReleaseKernel(kernel); if (program) clReleaseProgram(program); - if (a_memobj) clReleaseMemObject(a_memobj); - if (b_memobj) clReleaseMemObject(b_memobj); - if (c_memobj) clReleaseMemObject(c_memobj); + if (i_memobj) clReleaseMemObject(i_memobj); + if (w_memobj) clReleaseMemObject(w_memobj); + if (o_memobj) clReleaseMemObject(o_memobj); if (context) clReleaseContext(context); if (device_id) clReleaseDevice(device_id); if (kernel_bin) free(kernel_bin); } -int size = 64; +int size = 32; static void show_usage() { printf("Usage: [-n size] [-h: help]\n"); @@ -127,7 +137,7 @@ static void show_usage() { static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "fn:h?")) != -1) { + while ((c = getopt(argc, argv, "n:h?")) != -1) { switch (c) { case 'n': size = atoi(optarg); @@ -149,10 +159,10 @@ int main (int argc, char **argv) { parse_args(argc, argv); printf("Matrix size=%d\n", size); - if ((size / LOCAL_SIZE) * LOCAL_SIZE != size) { - printf("Error: matrix size must be a multiple of %d\n", LOCAL_SIZE); - return -1; - } + + uint32_t o_points = size * size; + uint32_t i_points = (size+2) * (size+2); + uint32_t w_points = 3 * 3; cl_platform_id platform_id; size_t kernel_size; @@ -168,11 +178,13 @@ int main (int argc, char **argv) { clGetDeviceInfo(device_id, CL_DEVICE_NAME, sizeof(device_string), &device_string, NULL); printf("Using device: %s\n", device_string); - printf("Allocate device buffers\n"); - size_t nbytes = size * size * sizeof(float); - a_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_READ_ONLY, nbytes, NULL, &_err)); - b_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_READ_ONLY, nbytes, NULL, &_err)); - c_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_WRITE_ONLY, nbytes, NULL, &_err)); + printf("Allocate device buffers\n"); + size_t i_nbytes = i_points * sizeof(float); + size_t w_nbytes = w_points * sizeof(float); + size_t o_nbytes = o_points * sizeof(float); + i_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_READ_ONLY, i_nbytes, NULL, &_err)); + w_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_READ_ONLY, w_nbytes, NULL, &_err)); + o_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_WRITE_ONLY, o_nbytes, NULL, &_err)); printf("Create program from kernel source\n"); #ifdef HOSTGPU @@ -197,32 +209,32 @@ int main (int argc, char **argv) { // Create kernel kernel = CL_CHECK2(clCreateKernel(program, KERNEL_NAME, &_err)); - size_t local_size[2] = {LOCAL_SIZE, LOCAL_SIZE}; size_t global_size[2] = {size, size}; // Set kernel arguments - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), (void *)&a_memobj)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), (void *)&b_memobj)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), (void *)&c_memobj)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), (void *)&o_memobj)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), (void *)&i_memobj)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), (void *)&w_memobj)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(uint32_t), &size)); - CL_CHECK(clSetKernelArg(kernel, 4, local_size[0]*local_size[1]*sizeof(float), NULL)); - CL_CHECK(clSetKernelArg(kernel, 5, local_size[0]*local_size[1]*sizeof(float), NULL)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(uint32_t), &size)); - // Allocate memories for input arrays and output arrays. - std::vector h_a(size * size); - std::vector h_b(size * size); - std::vector h_c(size * size); + // Allocate memories for input arrays and output arrays. + std::vector h_i(i_points); + std::vector h_w(w_points); + std::vector h_o(o_points, 0.0f); - // Initialize values for array members. - for (int i = 0; i < (size * size); ++i) { - #ifdef USE_FLOAT - h_a[i] = (float)rand() / (float)RAND_MAX; - h_b[i] = (float)rand() / (float)RAND_MAX; - #else - h_a[i] = rand(); - h_b[i] = rand(); - #endif - h_c[i] = 0xdeadbeef; + // Generate input values + for (int32_t y = -1; y < size+1; ++y) { + for (int32_t x = -1; x < size+1; ++x) { + if (x >= 0 && x < size && y >= 0 && y < size) { + h_i[(y+1) * (size+2) + (x+1)] = static_cast(rand()) / RAND_MAX; + } else { + h_i[(y+1) * (size+2) + (x+1)] = 0; + } + } + } + for (uint32_t i = 0; i < w_points; ++i) { + h_w[i] = static_cast(rand()) / RAND_MAX; } // NOTE(hansung): Dump operand buffer to a file @@ -235,28 +247,28 @@ int main (int argc, char **argv) { commandQueue = CL_CHECK2(clCreateCommandQueue(context, device_id, 0, &_err)); printf("Upload source buffers\n"); - CL_CHECK(clEnqueueWriteBuffer(commandQueue, a_memobj, CL_TRUE, 0, nbytes, h_a.data(), 0, NULL, NULL)); - CL_CHECK(clEnqueueWriteBuffer(commandQueue, b_memobj, CL_TRUE, 0, nbytes, h_b.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueWriteBuffer(commandQueue, i_memobj, CL_TRUE, 0, i_nbytes, h_i.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueWriteBuffer(commandQueue, w_memobj, CL_TRUE, 0, w_nbytes, h_w.data(), 0, NULL, NULL)); printf("Execute the kernel\n"); auto time_start = std::chrono::high_resolution_clock::now(); - CL_CHECK(clEnqueueNDRangeKernel(commandQueue, kernel, 2, NULL, global_size, local_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueNDRangeKernel(commandQueue, kernel, 2, NULL, global_size, NULL, 0, NULL, NULL)); CL_CHECK(clFinish(commandQueue)); auto time_end = std::chrono::high_resolution_clock::now(); double elapsed = std::chrono::duration_cast(time_end - time_start).count(); printf("Elapsed time: %lg ms\n", elapsed); printf("Download destination buffer\n"); - CL_CHECK(clEnqueueReadBuffer(commandQueue, c_memobj, CL_TRUE, 0, nbytes, h_c.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(commandQueue, o_memobj, CL_TRUE, 0, o_nbytes, h_o.data(), 0, NULL, NULL)); printf("Verify result\n"); - std::vector ref_vec(size * size); - matrix_multiply_cpu(h_a.data(), h_b.data(), ref_vec.data(), size); + std::vector ref_vec(o_points); + convolution_cpu(ref_vec.data(), h_i.data(), h_w.data(), size, size); int errors = 0; - for (int i = 0; i < (size * size); i++) { - if (!compare_equal(h_c[i], ref_vec[i])) { + for (uint32_t i = 0; i < o_points; ++i) { + if (!compare_equal(h_o[i], ref_vec[i])) { if (errors < 100) - printf("*** error: [%d] expected=%f, actual=%f\n", i, ref_vec[i], h_c[i]); + printf("*** error: [%d] expected=%f, actual=%f\n", i, ref_vec[i], h_o[i]); ++errors; } } diff --git a/tests/opencl/cutcp/Makefile b/tests/opencl/cutcp/Makefile deleted file mode 100644 index d16ddb11..00000000 --- a/tests/opencl/cutcp/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -PROJECT = cutcp - -SRCS = main.cc args.c parboil_opencl.c ocl.c gpu_info.c cutoff.c cutcpu.c output.c readatom.c excl.c - -CXXFLAGS += -I. - -OPTS ?= - -include ../common.mk diff --git a/tests/opencl/cutcp/args.c b/tests/opencl/cutcp/args.c deleted file mode 100644 index 9d751e29..00000000 --- a/tests/opencl/cutcp/args.c +++ /dev/null @@ -1,617 +0,0 @@ - -#include -#include -#include -#include -#include -#include - -/*****************************************************************************/ -/* Memory management routines */ - -/* Free an array of owned strings. */ -void -pb_FreeStringArray(char **string_array) -{ - char **p; - - if (!string_array) return; - for (p = string_array; *p; p++) free(*p); - free(string_array); -} - -struct pb_PlatformParam * -pb_PlatformParam(char *name, char *version) -{ - if (name == NULL) { - fprintf(stderr, "pb_PlatformParam: Invalid argument\n"); - exit(-1); - } - - struct pb_PlatformParam *ret = - (struct pb_PlatformParam *)malloc(sizeof (struct pb_PlatformParam)); - - ret->name = name; - ret->version = version; - return ret; -} - -void -pb_FreePlatformParam(struct pb_PlatformParam *p) -{ - if (p == NULL) return; - - free(p->name); - free(p->version); - free(p); -} - -struct pb_DeviceParam * -pb_DeviceParam_index(int index) -{ - struct pb_DeviceParam *ret = - (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); - ret->criterion = pb_Device_INDEX; - ret->index = index; - return ret; -} - -struct pb_DeviceParam * -pb_DeviceParam_cpu(void) -{ - struct pb_DeviceParam *ret = - (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); - ret->criterion = pb_Device_CPU; - return ret; -} - -struct pb_DeviceParam * -pb_DeviceParam_gpu(void) -{ - struct pb_DeviceParam *ret = - (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); - ret->criterion = pb_Device_GPU; - return ret; -} - -struct pb_DeviceParam * -pb_DeviceParam_accelerator(void) -{ - struct pb_DeviceParam *ret = - (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); - ret->criterion = pb_Device_ACCELERATOR; - return ret; -} - -struct pb_DeviceParam * -pb_DeviceParam_name(char *name) -{ - struct pb_DeviceParam *ret = - (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); - ret->criterion = pb_Device_NAME; - ret->name = name; - return ret; -} - -void -pb_FreeDeviceParam(struct pb_DeviceParam *p) -{ - if (p == NULL) return; - - switch(p->criterion) { - case pb_Device_NAME: - free(p->name); - break; - case pb_Device_INDEX: - case pb_Device_CPU: - case pb_Device_ACCELERATOR: - break; - default: - fprintf(stderr, "pb_FreeDeviceParam: Invalid argument\n"); - exit(-1); - } -} - -void -pb_FreeParameters(struct pb_Parameters *p) -{ - free(p->outFile); - pb_FreeStringArray(p->inpFiles); - pb_FreePlatformParam(p->platform); - pb_FreeDeviceParam(p->device); - free(p); -} - -/*****************************************************************************/ - -/* Parse a comma-delimited list of strings into an - * array of strings. */ -static char ** -read_string_array(char *in) -{ - char **ret; - int i; - int count; /* Number of items in the input */ - char *substring; /* Current substring within 'in' */ - - /* Count the number of items in the string */ - count = 1; - for (i = 0; in[i]; i++) if (in[i] == ',') count++; - - /* Allocate storage */ - ret = (char **)malloc((count + 1) * sizeof(char *)); - - /* Create copies of the strings from the list */ - substring = in; - for (i = 0; i < count; i++) { - char *substring_end; - int substring_length; - - /* Find length of substring */ - for (substring_end = substring; - (*substring_end != ',') && (*substring_end != 0); - substring_end++); - - substring_length = substring_end - substring; - - /* Allocate memory and copy the substring */ - ret[i] = (char *)malloc(substring_length + 1); - memcpy(ret[i], substring, substring_length); - ret[i][substring_length] = 0; - - /* go to next substring */ - substring = substring_end + 1; - } - ret[i] = NULL; /* Write the sentinel value */ - - return ret; -} - -static void -report_parse_error(const char *str) -{ - fputs(str, stderr); -} - -/* Interpret a string as a 'pb_DeviceParam' value. - * Return a pointer to a new value, or NULL on failure. - */ -static struct pb_DeviceParam * -read_device_param(char *str) -{ - /* Try different ways of interpreting 'device_string' until one works */ - - /* If argument is an integer, then interpret it as a device index */ - errno = 0; - char *end; - long device_int = strtol(str, &end, 10); - if (!errno) { - /* Negative numbers are not valid */ - if (device_int < 0 || device_int > INT_MAX) return NULL; - - return pb_DeviceParam_index(device_int); - } - - /* Match against predefined strings */ - if (strcmp(str, "CPU") == 0) - return pb_DeviceParam_cpu(); - if (strcmp(str, "GPU") == 0) - return pb_DeviceParam_gpu(); - if (strcmp(str, "ACCELERATOR") == 0) - return pb_DeviceParam_accelerator(); - - /* Assume any other string is a device name */ - return pb_DeviceParam_name(strdup(str)); -} - -/* Interpret a string as a 'pb_PlatformParam' value. - * Return a pointer to a new value, or NULL on failure. - */ -static struct pb_PlatformParam * -read_platform_param(char *str) -{ - int separator_index; /* Index of the '-' character separating - * name and version number. It's -1 if - * there's no '-' character. */ - - /* Find the last occurrence of '-' in 'str' */ - { - char *cur; - separator_index = -1; - for (cur = str; *cur; cur++) { - if (*cur == '-') separator_index = cur - str; - } - } - - /* The platform name is either the entire string, or all characters before - * the separator */ - int name_length = separator_index == -1 ? strlen(str) : separator_index; - char *name_str = (char *)malloc(name_length + 1); - memcpy(name_str, str, name_length); - name_str[name_length] = 0; - - /* The version is either NULL, or all characters after the separator */ - char *version_str; - if (separator_index == -1) { - version_str = NULL; - } - else { - const char *version_input_str = str + separator_index + 1; - int version_length = strlen(version_input_str); - - version_str = (char *)malloc(version_length + 1); - memcpy(version_str, version_input_str, version_length); - version_str[version_length] = 0; - } - - /* Create output structure */ - return pb_PlatformParam(name_str, version_str); -} - -/****************************************************************************/ -/* Argument parsing state */ - -/* Argument parsing state. - * - * Arguments that are interpreted by the argument parser are removed from - * the list. Variables 'argc' and 'argn' do not count arguments that have - * been removed. - * - * During argument parsing, the array of arguments is compacted, overwriting - * the erased arguments. Variable 'argv_put' points to the array element - * where the next argument will be written. Variable 'argv_get' points to - * the array element where the next argument will be read from. - */ -struct argparse { - int argc; /* Number of arguments. Mutable. */ - int argn; /* Current argument index. */ - char **argv_get; /* Argument value being read. */ - char **argv_put; /* Argument value being written. - * argv_put <= argv_get. */ -}; - -static void -initialize_argparse(struct argparse *ap, int argc, char **argv) -{ - ap->argc = argc; - ap->argn = 0; - ap->argv_get = ap->argv_put = argv; -} - -/* Finish argument parsing, without processing the remaining arguments. - * Write new argument count into _argc. */ -static void -finalize_argparse(struct argparse *ap, int *_argc, char **argv) -{ - /* Move the remaining arguments */ - for(; ap->argn < ap->argc; ap->argn++) - *ap->argv_put++ = *ap->argv_get++; - - /* Update the argument count */ - *_argc = ap->argc; - - /* Insert a terminating NULL */ - argv[ap->argc] = NULL; -} - -/* Delete the current argument. The argument will not be visible - * when argument parsing is done. */ -static void -delete_argument(struct argparse *ap) -{ - if (ap->argn >= ap->argc) { - fprintf(stderr, "delete_argument\n"); - } - ap->argc--; - ap->argv_get++; -} - -/* Go to the next argument. Also, move the current argument to its - * final location in argv. */ -static void -next_argument(struct argparse *ap) -{ - if (ap->argn >= ap->argc) { - fprintf(stderr, "next_argument\n"); - } - /* Move argument to its new location. */ - *ap->argv_put++ = *ap->argv_get++; - ap->argn++; -} - -static int -is_end_of_arguments(struct argparse *ap) -{ - return ap->argn == ap->argc; -} - -/* Get the current argument */ -static char * -get_argument(struct argparse *ap) -{ - return *ap->argv_get; -} - -/* Get the current argument, and also delete it */ -static char * -consume_argument(struct argparse *ap) -{ - char *ret = get_argument(ap); - delete_argument(ap); - return ret; -} - -/****************************************************************************/ - -/* The result of parsing a command-line argument */ -typedef enum { - ARGPARSE_OK, /* Success */ - ARGPARSE_ERROR, /* Error */ - ARGPARSE_DONE /* Success, and do not continue parsing */ -} result; - -typedef result parse_action(struct argparse *ap, struct pb_Parameters *params); - - -/* A command-line option */ -struct option { - char short_name; /* If not 0, the one-character - * name of this option */ - const char *long_name; /* If not NULL, the long name of this option */ - parse_action *action; /* What to do when this option occurs. - * Sentinel value is NULL. - */ -}; - -/* Output file - * - * -o FILE - */ -static result -parse_output_file(struct argparse *ap, struct pb_Parameters *params) -{ - if (is_end_of_arguments(ap)) - { - report_parse_error("Expecting file name after '-o'\n"); - return ARGPARSE_ERROR; - } - - /* Replace the output file name */ - free(params->outFile); - params->outFile = strdup(consume_argument(ap)); - - return ARGPARSE_OK; -} - -/* Input files - * - * -i FILE,FILE,... - */ -static result -parse_input_files(struct argparse *ap, struct pb_Parameters *params) -{ - if (is_end_of_arguments(ap)) - { - report_parse_error("Expecting file name after '-i'\n"); - return ARGPARSE_ERROR; - } - - /* Replace the input file list */ - pb_FreeStringArray(params->inpFiles); - params->inpFiles = read_string_array(consume_argument(ap)); - return ARGPARSE_OK; -} - -/* End of options - * - * -- - */ - -static result -parse_end_options(struct argparse *ap, struct pb_Parameters *params) -{ - return ARGPARSE_DONE; -} - -/* OpenCL device - * - * --device X - */ - -static result -parse_device(struct argparse *ap, struct pb_Parameters *params) -{ - /* Read the next argument, which specifies a device */ - - if (is_end_of_arguments(ap)) - { - report_parse_error("Expecting device specification after '--device'\n"); - return ARGPARSE_ERROR; - } - - char *device_string = consume_argument(ap); - struct pb_DeviceParam *device_param = read_device_param(device_string); - - if (!device_param) { - report_parse_error("Unrecognized device specification format on command line\n"); - return ARGPARSE_ERROR; - } - - /* Save the result */ - pb_FreeDeviceParam(params->device); - params->device = device_param; - - return ARGPARSE_OK; -} - -static result -parse_platform(struct argparse *ap, struct pb_Parameters *params) -{ - /* Read the next argument, which specifies a platform */ - - if (is_end_of_arguments(ap)) - { - report_parse_error("Expecting device specification after '--platform'\n"); - return ARGPARSE_ERROR; - } - - char *platform_string = consume_argument(ap); - struct pb_PlatformParam *platform_param = read_platform_param(platform_string); - - if (!platform_param) { - report_parse_error("Unrecognized platform specification format on command line\n"); - return ARGPARSE_ERROR; - } - - /* Save the result */ - pb_FreePlatformParam(params->platform); - params->platform = platform_param; - - return ARGPARSE_OK; -} - - -static struct option options[] = { - { 'o', NULL, &parse_output_file }, - { 'i', NULL, &parse_input_files }, - { '-', NULL, &parse_end_options }, - { 0, "device", &parse_device }, - { 0, "platform", &parse_platform }, - { 0, NULL, NULL } -}; - -static int -is_last_option(struct option *op) -{ - return op->action == NULL; -} - -/****************************************************************************/ - -/* Parse command-line parameters. - * Return zero on error, nonzero otherwise. - * On error, the other outputs may be invalid. - * - * The information collected from parameters is used to update - * 'ret'. 'ret' should be initialized. - * - * '_argc' and 'argv' are updated to contain only the unprocessed arguments. - */ -static int -pb_ParseParameters (struct pb_Parameters *ret, int *_argc, char **argv) -{ - char *err_message; - struct argparse ap; - - /* Each argument */ - initialize_argparse(&ap, *_argc, argv); - while(!is_end_of_arguments(&ap)) { - result arg_result; /* Result of parsing this option */ - char *arg = get_argument(&ap); - - /* Process this argument */ - if (arg[0] == '-') { - /* Single-character flag */ - if ((arg[1] != 0) && (arg[2] == 0)) { - delete_argument(&ap); /* This argument is consumed here */ - - /* Find a matching short option */ - struct option *op; - for (op = options; !is_last_option(op); op++) { - if (op->short_name == arg[1]) { - arg_result = (*op->action)(&ap, ret); - goto option_was_processed; - } - } - - /* No option matches */ - report_parse_error("Unexpected command-line parameter\n"); - arg_result = ARGPARSE_ERROR; - goto option_was_processed; - } - - /* Long flag */ - if (arg[1] == '-') { - delete_argument(&ap); /* This argument is consumed here */ - - /* Find a matching long option */ - struct option *op; - for (op = options; !is_last_option(op); op++) { - if (op->long_name && strcmp(&arg[2], op->long_name) == 0) { - arg_result = (*op->action)(&ap, ret); - goto option_was_processed; - } - } - - /* No option matches */ - report_parse_error("Unexpected command-line parameter\n"); - arg_result = ARGPARSE_ERROR; - goto option_was_processed; - } - } - else { - /* Other arguments are ignored */ - next_argument(&ap); - arg_result = ARGPARSE_OK; - goto option_was_processed; - } - - option_was_processed: - /* Decide what to do next based on 'arg_result' */ - switch(arg_result) { - case ARGPARSE_OK: - /* Continue processing */ - break; - - case ARGPARSE_ERROR: - /* Error exit from the function */ - return 0; - - case ARGPARSE_DONE: - /* Normal exit from the argument parsing loop */ - goto end_of_options; - } - } /* end for each argument */ - - /* If all arguments were processed, then normal exit from the loop */ - - end_of_options: - finalize_argparse(&ap, _argc, argv); - return 1; -} - -/*****************************************************************************/ -/* Other exported functions */ - -struct pb_Parameters * -pb_ReadParameters(int *_argc, char **argv) -{ - struct pb_Parameters *ret = - (struct pb_Parameters *)malloc(sizeof(struct pb_Parameters)); - - /* Initialize the parameters structure */ - ret->outFile = NULL; - ret->inpFiles = (char **)malloc(sizeof(char *)); - ret->inpFiles[0] = NULL; - ret->platform = NULL; - ret->device = NULL; - - /* Read parameters and update _argc, argv */ - if (!pb_ParseParameters(ret, _argc, argv)) { - /* Parse error */ - pb_FreeParameters(ret); - return NULL; - } - - return ret; -} - -int -pb_Parameters_CountInputs(struct pb_Parameters *p) -{ - int n; - - for (n = 0; p->inpFiles[n]; n++); - return n; -} - diff --git a/tests/opencl/cutcp/atom.h b/tests/opencl/cutcp/atom.h deleted file mode 100644 index f5a60058..00000000 --- a/tests/opencl/cutcp/atom.h +++ /dev/null @@ -1,37 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2008-2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#ifndef ATOM_H -#define ATOM_H - -#ifdef __cplusplus -extern "C" { -#endif - - typedef struct Atom_t { - float x, y, z, q; - } Atom; - - typedef struct Atoms_t { - Atom *atoms; - int size; - } Atoms; - - typedef struct Vec3_t { - float x, y, z; - } Vec3; - - Atoms *read_atom_file(const char *fname); - void free_atom(Atoms *atom); - void get_atom_extent(Vec3 *lo, Vec3 *hi, Atoms *atom); - -#ifdef __cplusplus -} -#endif - -#endif /* ATOM_H */ diff --git a/tests/opencl/cutcp/cutcpu.c b/tests/opencl/cutcp/cutcpu.c deleted file mode 100644 index 5f440752..00000000 --- a/tests/opencl/cutcp/cutcpu.c +++ /dev/null @@ -1,195 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2008-2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#include -#include -#include -#include -#include -#include "atom.h" -#include "cutoff.h" - -#undef DEBUG_PASS_RATE -#define CHECK_CYLINDER_CPU - -#define CELLEN 4.f -#define INV_CELLEN (1.f/CELLEN) - -extern int cpu_compute_cutoff_potential_lattice( - Lattice *lattice, /* the lattice */ - float cutoff, /* cutoff distance */ - Atoms *atoms /* array of atoms */ - ) -{ - int nx = lattice->dim.nx; - int ny = lattice->dim.ny; - int nz = lattice->dim.nz; - float xlo = lattice->dim.lo.x; - float ylo = lattice->dim.lo.y; - float zlo = lattice->dim.lo.z; - float gridspacing = lattice->dim.h; - int natoms = atoms->size; - Atom *atom = atoms->atoms; - - const float a2 = cutoff * cutoff; - const float inv_a2 = 1.f / a2; - float s; - const float inv_gridspacing = 1.f / gridspacing; - const int radius = (int) ceilf(cutoff * inv_gridspacing) - 1; - /* lattice point radius about each atom */ - - int n; - int i, j, k; - int ia, ib, ic; - int ja, jb, jc; - int ka, kb, kc; - int index; - int koff, jkoff; - - float x, y, z, q; - float dx, dy, dz; - float dz2, dydz2, r2; - float e; - float xstart, ystart; - - float *pg; - - int gindex; - int ncell, nxcell, nycell, nzcell; - int *first, *next; - float inv_cellen = INV_CELLEN; - Vec3 minext, maxext; /* Extent of atom bounding box */ - float xmin, ymin, zmin; - float xmax, ymax, zmax; - -#if DEBUG_PASS_RATE - unsigned long long pass_count = 0; - unsigned long long fail_count = 0; -#endif - - /* find min and max extent */ - get_atom_extent(&minext, &maxext, atoms); - - /* number of cells in each dimension */ - nxcell = (int) floorf((maxext.x-minext.x) * inv_cellen) + 1; - nycell = (int) floorf((maxext.y-minext.y) * inv_cellen) + 1; - nzcell = (int) floorf((maxext.z-minext.z) * inv_cellen) + 1; - ncell = nxcell * nycell * nzcell; - - /* allocate for cursor link list implementation */ - first = (int *) malloc(ncell * sizeof(int)); - for (gindex = 0; gindex < ncell; gindex++) { - first[gindex] = -1; - } - next = (int *) malloc(natoms * sizeof(int)); - for (n = 0; n < natoms; n++) { - next[n] = -1; - } - - /* geometric hashing */ - for (n = 0; n < natoms; n++) { - if (0==atom[n].q) continue; /* skip any non-contributing atoms */ - i = (int) floorf((atom[n].x - minext.x) * inv_cellen); - j = (int) floorf((atom[n].y - minext.y) * inv_cellen); - k = (int) floorf((atom[n].z - minext.z) * inv_cellen); - gindex = (k*nycell + j)*nxcell + i; - next[n] = first[gindex]; - first[gindex] = n; - } - - /* traverse the grid cells */ - for (gindex = 0; gindex < ncell; gindex++) { - for (n = first[gindex]; n != -1; n = next[n]) { - x = atom[n].x - xlo; - y = atom[n].y - ylo; - z = atom[n].z - zlo; - q = atom[n].q; - - /* find closest grid point with position less than or equal to atom */ - ic = (int) (x * inv_gridspacing); - jc = (int) (y * inv_gridspacing); - kc = (int) (z * inv_gridspacing); - - /* find extent of surrounding box of grid points */ - ia = ic - radius; - ib = ic + radius + 1; - ja = jc - radius; - jb = jc + radius + 1; - ka = kc - radius; - kb = kc + radius + 1; - - /* trim box edges so that they are within grid point lattice */ - if (ia < 0) ia = 0; - if (ib >= nx) ib = nx-1; - if (ja < 0) ja = 0; - if (jb >= ny) jb = ny-1; - if (ka < 0) ka = 0; - if (kb >= nz) kb = nz-1; - - /* loop over surrounding grid points */ - xstart = ia*gridspacing - x; - ystart = ja*gridspacing - y; - dz = ka*gridspacing - z; - for (k = ka; k <= kb; k++, dz += gridspacing) { - koff = k*ny; - dz2 = dz*dz; - dy = ystart; - for (j = ja; j <= jb; j++, dy += gridspacing) { - jkoff = (koff + j)*nx; - dydz2 = dy*dy + dz2; -#ifdef CHECK_CYLINDER_CPU - if (dydz2 >= a2) continue; -#endif - - dx = xstart; - index = jkoff + ia; - pg = lattice->lattice + index; - -#if defined(__INTEL_COMPILER) - for (i = ia; i <= ib; i++, pg++, dx += gridspacing) { - r2 = dx*dx + dydz2; - s = (1.f - r2 * inv_a2) * (1.f - r2 * inv_a2); - e = q * (1/sqrtf(r2)) * s; - *pg += (r2 < a2 ? e : 0); /* LOOP VECTORIZED!! */ - } -#else - for (i = ia; i <= ib; i++, pg++, dx += gridspacing) { - r2 = dx*dx + dydz2; - if (r2 >= a2) - { -#ifdef DEBUG_PASS_RATE - fail_count++; -#endif - continue; - } -#ifdef DEBUG_PASS_RATE - pass_count++; -#endif - s = (1.f - r2 * inv_a2); - e = q * (1/sqrtf(r2)) * s * s; - *pg += e; - } -#endif - } - } /* end loop over surrounding grid points */ - - } /* end loop over atoms in a gridcell */ - } /* end loop over gridcells */ - - /* free memory */ - free(next); - free(first); - - /* For debugging: print the number of times that the test passed/failed */ -#ifdef DEBUG_PASS_RATE - printf ("Pass :%lld\n", pass_count); - printf ("Fail :%lld\n", fail_count); -#endif - - return 0; -} diff --git a/tests/opencl/cutcp/cutoff.c b/tests/opencl/cutcp/cutoff.c deleted file mode 100644 index 4a52d528..00000000 --- a/tests/opencl/cutcp/cutoff.c +++ /dev/null @@ -1,508 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2008-2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#include - -#include -#include -#include -#include -#include - -#include "atom.h" -#include "cutoff.h" -#include "macros.h" -#include "ocl.h" - -static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) { - if (NULL == filename || NULL == data || 0 == size) - return CL_INVALID_VALUE; - - FILE* fp = fopen(filename, "r"); - if (NULL == fp) { - fprintf(stderr, "Failed to load kernel."); - return CL_INVALID_VALUE; - } - fseek(fp , 0 , SEEK_END); - long fsize = ftell(fp); - rewind(fp); - - *data = (uint8_t*)malloc(fsize); - *size = fread(*data, 1, fsize, fp); - - fclose(fp); - - return CL_SUCCESS; -} - -// OpenCL 1.1 support for int3 is not uniform on all implementations, so -// we use int4 instead. Only the 'x', 'y', and 'z' fields of xyz are used. -typedef cl_int4 xyz; - -//extern "C" int gpu_compute_cutoff_potential_lattice( -int gpu_compute_cutoff_potential_lattice( - struct pb_TimerSet *timers, - Lattice *lattice, /* the lattice */ - float cutoff, /* cutoff distance */ - Atoms *atoms, /* array of atoms */ - int verbose, /* print info/debug messages */ - struct pb_Parameters *parameters - ) -{ - int nx = lattice->dim.nx; - int ny = lattice->dim.ny; - int nz = lattice->dim.nz; - float xlo = lattice->dim.lo.x; - float ylo = lattice->dim.lo.y; - float zlo = lattice->dim.lo.z; - float h = lattice->dim.h; - int natoms = atoms->size; - Atom *atom = atoms->atoms; - - xyz nbrlist[NBRLIST_MAXLEN]; - int nbrlistlen = 0; - - int binHistoFull[BIN_DEPTH+1] = { 0 }; /* clear every array element */ - int binHistoCover[BIN_DEPTH+1] = { 0 }; /* clear every array element */ - int num_excluded = 0; - - int xRegionDim, yRegionDim, zRegionDim; - int xRegionIndex, yRegionIndex, zRegionIndex; - int xOffset, yOffset, zOffset; - int lnx, lny, lnz, lnall; - float *regionZeroAddr, *thisRegion; - cl_mem regionZeroCl; - int index, indexRegion; - - int c; - xyz binDim; - int nbins; - cl_float4 *binBaseAddr, *binZeroAddr; - cl_mem binBaseCl, binZeroCl; - int *bincntBaseAddr, *bincntZeroAddr; - Atoms *extra = NULL; - - cl_mem NbrListLen; - cl_mem NbrList; - - int i, j, k, n; - int sum, total; - - float avgFillFull, avgFillCover; - const float cutoff2 = cutoff * cutoff; - const float inv_cutoff2 = 1.f / cutoff2; - - size_t gridDim[3], blockDim[3]; - - // The "compute" timer should be active upon entry to this function - - /* pad lattice to be factor of 8 in each dimension */ - xRegionDim = (int) ceilf(nx/8.f); - yRegionDim = (int) ceilf(ny/8.f); - zRegionDim = (int) ceilf(nz/8.f); - - lnx = 8 * xRegionDim; - lny = 8 * yRegionDim; - lnz = 8 * zRegionDim; - lnall = lnx * lny * lnz; - - /* will receive energies from OpenCL */ - regionZeroAddr = (float *) malloc(lnall * sizeof(float)); - - /* create bins */ - c = (int) ceil(cutoff * BIN_INVLEN); /* count extra bins around lattice */ - binDim.x = (int) ceil(lnx * h * BIN_INVLEN) + 2*c; - binDim.y = (int) ceil(lny * h * BIN_INVLEN) + 2*c; - binDim.z = (int) ceil(lnz * h * BIN_INVLEN) + 2*c; - nbins = binDim.x * binDim.y * binDim.z; - binBaseAddr = (cl_float4 *) calloc(nbins * BIN_DEPTH, sizeof(cl_float4)); - binZeroAddr = binBaseAddr + ((c * binDim.y + c) * binDim.x + c) * BIN_DEPTH; - - bincntBaseAddr = (int *) calloc(nbins, sizeof(int)); - bincntZeroAddr = bincntBaseAddr + (c * binDim.y + c) * binDim.x + c; - - /* create neighbor list */ - if (ceilf(BIN_LENGTH / (8*h)) == floorf(BIN_LENGTH / (8*h))) { - float s = sqrtf(3); - float r2 = (cutoff + s*BIN_LENGTH) * (cutoff + s*BIN_LENGTH); - int cnt = 0; - /* develop neighbor list around 1 cell */ - if (2*c + 1 > NBRLIST_DIM) { - fprintf(stderr, "must have cutoff <= %f\n", - (NBRLIST_DIM-1)/2 * BIN_LENGTH); - return -1; - } - for (k = -c; k <= c; k++) { - for (j = -c; j <= c; j++) { - for (i = -c; i <= c; i++) { - if ((i*i + j*j + k*k)*BIN_LENGTH*BIN_LENGTH >= r2) continue; - nbrlist[cnt].x = i; - nbrlist[cnt].y = j; - nbrlist[cnt].z = k; - cnt++; - } - } - } - nbrlistlen = cnt; - } - else if (8*h <= 2*BIN_LENGTH) { - float s = 2.f*sqrtf(3); - float r2 = (cutoff + s*BIN_LENGTH) * (cutoff + s*BIN_LENGTH); - int cnt = 0; - /* develop neighbor list around 3-cube of cells */ - if (2*c + 3 > NBRLIST_DIM) { - fprintf(stderr, "must have cutoff <= %f\n", - (NBRLIST_DIM-3)/2 * BIN_LENGTH); - return -1; - } - for (k = -c; k <= c; k++) { - for (j = -c; j <= c; j++) { - for (i = -c; i <= c; i++) { - if ((i*i + j*j + k*k)*BIN_LENGTH*BIN_LENGTH >= r2) continue; - nbrlist[cnt].x = i; - nbrlist[cnt].y = j; - nbrlist[cnt].z = k; - cnt++; - } - } - } - nbrlistlen = cnt; - } - else { - fprintf(stderr, "must have h <= %f\n", 0.25 * BIN_LENGTH); - return -1; - } - - /* perform geometric hashing of atoms into bins */ - { - /* array of extra atoms, permit average of one extra per bin */ - Atom *extra_atoms = (Atom *) calloc(nbins, sizeof(Atom)); - int extra_len = 0; - - for (n = 0; n < natoms; n++) { - cl_float4 p; - p.x = atom[n].x - xlo; - p.y = atom[n].y - ylo; - p.z = atom[n].z - zlo; - p.w = atom[n].q; - i = (int) floorf(p.x * BIN_INVLEN); - j = (int) floorf(p.y * BIN_INVLEN); - k = (int) floorf(p.z * BIN_INVLEN); - if (i >= -c && i < binDim.x - c && - j >= -c && j < binDim.y - c && - k >= -c && k < binDim.z - c && - atom[n].q != 0) { - int index = (k * binDim.y + j) * binDim.x + i; - cl_float4 *bin = binZeroAddr + index * BIN_DEPTH; - int bindex = bincntZeroAddr[index]; - if (bindex < BIN_DEPTH) { - /* copy atom into bin and increase counter for this bin */ - bin[bindex] = p; - bincntZeroAddr[index]++; - } - else { - /* add index to array of extra atoms to be computed with CPU */ - if (extra_len >= nbins) { - fprintf(stderr, "exceeded space for storing extra atoms\n"); - return -1; - } - extra_atoms[extra_len] = atom[n]; - extra_len++; - } - } - else { - /* excluded atoms are either outside bins or neutrally charged */ - num_excluded++; - } - } - - /* Save result */ - extra = (Atoms *)malloc(sizeof(Atoms)); - extra->atoms = extra_atoms; - extra->size = extra_len; - } - - /* bin stats */ - sum = total = 0; - for (n = 0; n < nbins; n++) { - binHistoFull[ bincntBaseAddr[n] ]++; - sum += bincntBaseAddr[n]; - total += BIN_DEPTH; - } - avgFillFull = sum / (float) total; - sum = total = 0; - for (k = 0; k < binDim.z - 2*c; k++) { - for (j = 0; j < binDim.y - 2*c; j++) { - for (i = 0; i < binDim.x - 2*c; i++) { - int index = (k * binDim.y + j) * binDim.x + i; - binHistoCover[ bincntZeroAddr[index] ]++; - sum += bincntZeroAddr[index]; - total += BIN_DEPTH; - } - } - } - avgFillCover = sum / (float) total; - - if (verbose) { - /* report */ - printf("number of atoms = %d\n", natoms); - printf("lattice spacing = %g\n", h); - printf("cutoff distance = %g\n", cutoff); - printf("\n"); - printf("requested lattice dimensions = %d %d %d\n", nx, ny, nz); - printf("requested space dimensions = %g %g %g\n", nx*h, ny*h, nz*h); - printf("expanded lattice dimensions = %d %d %d\n", lnx, lny, lnz); - printf("expanded space dimensions = %g %g %g\n", lnx*h, lny*h, lnz*h); - printf("number of bytes for lattice data = %u\n", (unsigned int) (lnall*sizeof(float))); - printf("\n"); - printf("bin padding thickness = %d\n", c); - printf("bin cover dimensions = %d %d %d\n", - binDim.x - 2*c, binDim.y - 2*c, binDim.z - 2*c); - printf("bin full dimensions = %d %d %d\n", binDim.x, binDim.y, binDim.z); - printf("number of bins = %d\n", nbins); - printf("total number of atom slots = %d\n", nbins * BIN_DEPTH); - printf("%% overhead space = %g\n", - (natoms / (double) (nbins * BIN_DEPTH)) * 100); - printf("number of bytes for bin data = %u\n", - (unsigned int)(nbins * BIN_DEPTH * sizeof(cl_float4))); - printf("\n"); - printf("bin histogram with padding:\n"); - sum = 0; - for (n = 0; n <= BIN_DEPTH; n++) { - printf(" number of bins with %d atoms: %d\n", n, binHistoFull[n]); - sum += binHistoFull[n]; - } - printf(" total number of bins: %d\n", sum); - printf(" %% average fill: %g\n", avgFillFull * 100); - printf("\n"); - printf("bin histogram excluding padding:\n"); - sum = 0; - for (n = 0; n <= BIN_DEPTH; n++) { - printf(" number of bins with %d atoms: %d\n", n, binHistoCover[n]); - sum += binHistoCover[n]; - } - printf(" total number of bins: %d\n", sum); - printf(" %% average fill: %g\n", avgFillCover * 100); - printf("\n"); - printf("number of extra atoms = %d\n", extra->size); - printf("%% atoms that are extra = %g\n", (extra->size / (double) natoms) * 100); - printf("\n"); - - /* sanity check on bins */ - sum = 0; - for (n = 0; n <= BIN_DEPTH; n++) { - sum += n * binHistoFull[n]; - } - sum += extra->size + num_excluded; - printf("sanity check on bin histogram with edges: " - "sum + others = %d\n", sum); - sum = 0; - for (n = 0; n <= BIN_DEPTH; n++) { - sum += n * binHistoCover[n]; - } - sum += extra->size + num_excluded; - printf("sanity check on bin histogram excluding edges: " - "sum + others = %d\n", sum); - printf("\n"); - - /* neighbor list */ - printf("neighbor list length = %d\n", nbrlistlen); - printf("\n"); - } - - pb_Context* pb_context; - pb_context = pb_InitOpenCLContext(parameters); - if (pb_context == NULL) { - fprintf (stderr, "Error: No OpenCL platform/device can be found."); - return -1; - } - - cl_int clStatus; - cl_device_id clDevice = (cl_device_id) pb_context->clDeviceId; - cl_platform_id clPlatform = (cl_platform_id) pb_context->clPlatformId; - cl_context clContext = (cl_context) pb_context->clContext; - - cl_command_queue clCommandQueue = clCreateCommandQueue(clContext,clDevice,CL_QUEUE_PROFILING_ENABLE,&clStatus); - CHECK_ERROR("clCreateCommandQueue") - - pb_SetOpenCL(&clContext, &clCommandQueue); - - //const char* clSource[] = {readFile("src/opencl_base/kernel.cl")}; - //cl_program clProgram = clCreateProgramWithSource(clContext,1,clSource,NULL,&clStatus); - uint8_t *kernel_bin = NULL; - size_t kernel_size; - cl_int binary_status = 0; - clStatus = read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size); - CHECK_ERROR("read_kernel_file") - cl_program clProgram = clCreateProgramWithBinary( - clContext, 1, &clDevice, &kernel_size, (const uint8_t**)&kernel_bin, &binary_status, &clStatus); - CHECK_ERROR("clCreateProgramWithSource") - - char clOptions[50]; - sprintf(clOptions,"-I src/opencl_base"); //-cl-nv-verbose - - clStatus = clBuildProgram(clProgram,1,&clDevice,clOptions,NULL,NULL); - if (clStatus != CL_SUCCESS) { - size_t string_size = 0; - clGetProgramBuildInfo(clProgram, clDevice, CL_PROGRAM_BUILD_LOG, - 0, NULL, &string_size); - char* string = (char*)malloc(string_size*sizeof(char)); - clGetProgramBuildInfo(clProgram, clDevice, CL_PROGRAM_BUILD_LOG, - string_size, string, NULL); - puts(string); - } - - CHECK_ERROR("clBuildProgram") - - cl_kernel clKernel = clCreateKernel(clProgram,"opencl_cutoff_potential_lattice",&clStatus); - CHECK_ERROR("clCreateKernel") - - /* setup OpenCL kernel parameters */ - blockDim[0] = 8; - blockDim[1] = 8; - blockDim[2] = 2; - gridDim[0] = 4 * xRegionDim * blockDim[0]; - gridDim[1] = yRegionDim * blockDim[1]; - gridDim[2] = 1 * blockDim[2]; - - /* allocate and initialize memory on OpenCL device */ - pb_SwitchToTimer(timers, pb_TimerID_COPY); - if (verbose) { - printf("Allocating %.2fMB on OpenCL device for potentials\n", - lnall * sizeof(float) / (double) (1024*1024)); - } - - regionZeroCl = clCreateBuffer(clContext,CL_MEM_WRITE_ONLY,lnall*sizeof(float),NULL,&clStatus); - CHECK_ERROR("clCreateBuffer") - - // clMemSet(clCommandQueue,regionZeroCl,0,lnall*sizeof(float)); - - if (verbose) { - printf("Allocating %.2fMB on OpenCL device for atom bins\n", - nbins * BIN_DEPTH * sizeof(cl_float4) / (double) (1024*1024)); - } - - binBaseCl = clCreateBuffer(clContext,CL_MEM_READ_ONLY,nbins*BIN_DEPTH*sizeof(cl_float4),NULL,&clStatus); - CHECK_ERROR("clCreateBuffer") - - clStatus = clEnqueueWriteBuffer(clCommandQueue,binBaseCl,CL_TRUE,0,nbins*BIN_DEPTH*sizeof(cl_float4),binBaseAddr,0,NULL,NULL); - CHECK_ERROR("clEnqueueWriteBuffer") - - //Sub buffers are not supported in OpenCL v1.0 - int offset = ((c * binDim.y + c) * binDim.x + c) * BIN_DEPTH; - - NbrListLen = clCreateBuffer(clContext,CL_MEM_READ_ONLY,sizeof(int),NULL,&clStatus); - CHECK_ERROR("clCreateBuffer") - clStatus = clEnqueueWriteBuffer(clCommandQueue,NbrListLen,CL_TRUE,0,sizeof(int),&nbrlistlen,0,NULL,NULL); - CHECK_ERROR("clEnqueueWriteBuffer") - - NbrList = clCreateBuffer(clContext,CL_MEM_READ_ONLY,NBRLIST_MAXLEN*sizeof(xyz),NULL,&clStatus); - CHECK_ERROR("clCreateBuffer") - clStatus = clEnqueueWriteBuffer(clCommandQueue,NbrList,CL_TRUE,0,nbrlistlen*sizeof(xyz),nbrlist,0,NULL,NULL); - CHECK_ERROR("clEnqueueWriteBuffer") - - if (verbose) - printf("\n"); - - clStatus = clSetKernelArg(clKernel,0,sizeof(int),&(binDim.x)); - clStatus = clSetKernelArg(clKernel,1,sizeof(int),&(binDim.y)); - clStatus = clSetKernelArg(clKernel,2,sizeof(cl_mem),&binBaseCl); - clStatus = clSetKernelArg(clKernel,3,sizeof(int),&offset); - clStatus = clSetKernelArg(clKernel,4,sizeof(float),&h); - clStatus = clSetKernelArg(clKernel,5,sizeof(float),&cutoff2); - clStatus = clSetKernelArg(clKernel,6,sizeof(float),&inv_cutoff2); - clStatus = clSetKernelArg(clKernel,7,sizeof(cl_mem),®ionZeroCl); - clStatus = clSetKernelArg(clKernel,9,sizeof(cl_mem),&NbrListLen); - clStatus = clSetKernelArg(clKernel,10,sizeof(cl_mem),&NbrList); - CHECK_ERROR("clSetKernelArg") - - /* loop over z-dimension, invoke OpenCL kernel for each x-y plane */ - pb_SwitchToTimer(timers, pb_TimerID_KERNEL); - printf("Invoking OpenCL kernel on %d region planes...\n", zRegionDim); - for (zRegionIndex = 0; zRegionIndex < zRegionDim; zRegionIndex++) { - printf(" computing plane %d\r", zRegionIndex); - fflush(stdout); - - clStatus = clSetKernelArg(clKernel,8,sizeof(int),&zRegionIndex); - CHECK_ERROR("clSetKernelArg") - - clStatus = clEnqueueNDRangeKernel(clCommandQueue,clKernel,3,NULL,gridDim,blockDim,0,NULL,NULL); - - CHECK_ERROR("clEnqueueNDRangeKernel") - - clStatus = clFinish(clCommandQueue); - - CHECK_ERROR("clFinish") - } - - printf("Finished OpenCL kernel calls\n"); - - /* copy result regions from OpenCL device */ - pb_SwitchToTimer(timers, pb_TimerID_COPY); - clStatus = clEnqueueReadBuffer(clCommandQueue,regionZeroCl,CL_TRUE,0,lnall*sizeof(float),regionZeroAddr,0,NULL,NULL); - CHECK_ERROR("clEnqueueReadBuffer") - - /* free OpenCL memory allocations */ - clStatus = clReleaseMemObject(regionZeroCl); - clStatus = clReleaseMemObject(binBaseCl); - clStatus = clReleaseMemObject(NbrListLen); - clStatus = clReleaseMemObject(NbrList); - CHECK_ERROR("clReleaseMemObject") - - clStatus = clReleaseKernel(clKernel); - clStatus = clReleaseProgram(clProgram); - clStatus = clReleaseCommandQueue(clCommandQueue); - clStatus = clReleaseContext(clContext); - - //free((void*)clSource[0]); - - /* transpose regions back into lattice */ - pb_SwitchToTimer(timers, pb_TimerID_COMPUTE); - for (k = 0; k < nz; k++) { - zRegionIndex = (k >> 3); - zOffset = (k & 7); - - for (j = 0; j < ny; j++) { - yRegionIndex = (j >> 3); - yOffset = (j & 7); - - for (i = 0; i < nx; i++) { - xRegionIndex = (i >> 3); - xOffset = (i & 7); - - thisRegion = regionZeroAddr - + ((zRegionIndex * yRegionDim + yRegionIndex) * xRegionDim - + xRegionIndex) * REGION_SIZE; - - indexRegion = (zOffset * 8 + yOffset) * 8 + xOffset; - index = (k * ny + j) * nx + i; - - lattice->lattice[index] = thisRegion[indexRegion]; - } - } - } - - /* handle extra atoms */ - if (extra->size > 0) { - printf("computing extra atoms on CPU\n"); - if (cpu_compute_cutoff_potential_lattice(lattice, cutoff, extra)) { - fprintf(stderr, "cpu_compute_cutoff_potential_lattice() failed " - "for extra atoms\n"); - return -1; - } - printf("\n"); - } - - /* cleanup memory allocations */ - free(regionZeroAddr); - free(binBaseAddr); - free(bincntBaseAddr); - free_atom(extra); - - return 0; -} diff --git a/tests/opencl/cutcp/cutoff.h b/tests/opencl/cutcp/cutoff.h deleted file mode 100644 index 4db688ba..00000000 --- a/tests/opencl/cutcp/cutoff.h +++ /dev/null @@ -1,72 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2008-2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#ifndef CUTOFF_H -#define CUTOFF_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define SHIFTED - - /* A structure to record how points in 3D space map to array - elements. Array element (z, y, x) - where 0 <= x < nx, 0 <= y < ny, 0 <= z < nz - maps to coordinate (xlo, ylo, zlo) + h * (x, y, z). - */ - typedef struct LatticeDim_t { - /* Number of lattice points in x, y, z dimensions */ - int nx, ny, nz; - - /* Lowest corner of lattice */ - Vec3 lo; - - /* Lattice spacing */ - float h; - } LatticeDim; - - /* An electric potential field sampled on a regular grid. The - lattice size and grid point positions are specified by 'dim'. - */ - typedef struct Lattice_t { - LatticeDim dim; - float *lattice; - } Lattice; - - LatticeDim lattice_from_bounding_box(Vec3 lo, Vec3 hi, float h); - - Lattice *create_lattice(LatticeDim dim); - void destroy_lattice(Lattice *); - - int gpu_compute_cutoff_potential_lattice( - struct pb_TimerSet *timers, - Lattice *lattice, - float cutoff, /* cutoff distance */ - Atoms *atom, /* array of atoms */ - int verbose, /* print info/debug messages */ - struct pb_Parameters *parameters - ); - - int cpu_compute_cutoff_potential_lattice( - Lattice *lattice, /* the lattice */ - float cutoff, /* cutoff distance */ - Atoms *atoms /* array of atoms */ - ); - - int remove_exclusions( - Lattice *lattice, /* the lattice */ - float exclcutoff, /* exclusion cutoff distance */ - Atoms *atom /* array of atoms */ - ); - -#ifdef __cplusplus -} -#endif - -#endif /* CUTOFF_H */ diff --git a/tests/opencl/cutcp/excl.c b/tests/opencl/cutcp/excl.c deleted file mode 100644 index 1216854a..00000000 --- a/tests/opencl/cutcp/excl.c +++ /dev/null @@ -1,157 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2008-2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#include -#include -#include -#include -#include - -#include "atom.h" -#include "cutoff.h" - -#define CELLEN 4.f -#define INV_CELLEN (1.f/CELLEN) - -extern int remove_exclusions( - Lattice *lattice, /* the lattice */ - float cutoff, /* exclusion cutoff distance */ - Atoms *atoms /* array of atoms */ - ) -{ - int nx = lattice->dim.nx; - int ny = lattice->dim.ny; - int nz = lattice->dim.nz; - float xlo = lattice->dim.lo.x; - float ylo = lattice->dim.lo.y; - float zlo = lattice->dim.lo.z; - float gridspacing = lattice->dim.h; - Atom *atom = atoms->atoms; - - const float a2 = cutoff * cutoff; - const float inv_gridspacing = 1.f / gridspacing; - const int radius = (int) ceilf(cutoff * inv_gridspacing) - 1; - /* lattice point radius about each atom */ - - int n; - int i, j, k; - int ia, ib, ic; - int ja, jb, jc; - int ka, kb, kc; - int index; - int koff, jkoff; - - float x, y, z, q; - float dx, dy, dz; - float dz2, dydz2, r2; - float e; - float xstart, ystart; - - float *pg; - - int gindex; - int ncell, nxcell, nycell, nzcell; - int *first, *next; - float inv_cellen = INV_CELLEN; - Vec3 minext, maxext; - - /* find min and max extent */ - get_atom_extent(&minext, &maxext, atoms); - - /* number of cells in each dimension */ - nxcell = (int) floorf((maxext.x-minext.x) * inv_cellen) + 1; - nycell = (int) floorf((maxext.y-minext.y) * inv_cellen) + 1; - nzcell = (int) floorf((maxext.z-minext.z) * inv_cellen) + 1; - ncell = nxcell * nycell * nzcell; - - /* allocate for cursor link list implementation */ - first = (int *) malloc(ncell * sizeof(int)); - for (gindex = 0; gindex < ncell; gindex++) { - first[gindex] = -1; - } - next = (int *) malloc(atoms->size * sizeof(int)); - for (n = 0; n < atoms->size; n++) { - next[n] = -1; - } - - /* geometric hashing */ - for (n = 0; n < atoms->size; n++) { - if (0==atom[n].q) continue; /* skip any non-contributing atoms */ - i = (int) floorf((atom[n].x - minext.x) * inv_cellen); - j = (int) floorf((atom[n].y - minext.y) * inv_cellen); - k = (int) floorf((atom[n].z - minext.z) * inv_cellen); - gindex = (k*nycell + j)*nxcell + i; - next[n] = first[gindex]; - first[gindex] = n; - } - - /* traverse the grid cells */ - for (gindex = 0; gindex < ncell; gindex++) { - for (n = first[gindex]; n != -1; n = next[n]) { - x = atom[n].x - xlo; - y = atom[n].y - ylo; - z = atom[n].z - zlo; - q = atom[n].q; - - /* find closest grid point with position less than or equal to atom */ - ic = (int) (x * inv_gridspacing); - jc = (int) (y * inv_gridspacing); - kc = (int) (z * inv_gridspacing); - - /* find extent of surrounding box of grid points */ - ia = ic - radius; - ib = ic + radius + 1; - ja = jc - radius; - jb = jc + radius + 1; - ka = kc - radius; - kb = kc + radius + 1; - - /* trim box edges so that they are within grid point lattice */ - if (ia < 0) ia = 0; - if (ib >= nx) ib = nx-1; - if (ja < 0) ja = 0; - if (jb >= ny) jb = ny-1; - if (ka < 0) ka = 0; - if (kb >= nz) kb = nz-1; - - /* loop over surrounding grid points */ - xstart = ia*gridspacing - x; - ystart = ja*gridspacing - y; - dz = ka*gridspacing - z; - for (k = ka; k <= kb; k++, dz += gridspacing) { - koff = k*ny; - dz2 = dz*dz; - - dy = ystart; - for (j = ja; j <= jb; j++, dy += gridspacing) { - jkoff = (koff + j)*nx; - dydz2 = dy*dy + dz2; - - dx = xstart; - index = jkoff + ia; - pg = lattice->lattice + index; - - for (i = ia; i <= ib; i++, pg++, dx += gridspacing) { - r2 = dx*dx + dydz2; - - /* If atom and lattice point are too close, set the lattice value - * to zero */ - if (r2 < a2) *pg = 0; - } - } - } /* end loop over surrounding grid points */ - - } /* end loop over atoms in a gridcell */ - } /* end loop over gridcells */ - - /* free memory */ - free(next); - free(first); - - return 0; -} diff --git a/tests/opencl/cutcp/gpu_info.c b/tests/opencl/cutcp/gpu_info.c deleted file mode 100644 index 4d641f81..00000000 --- a/tests/opencl/cutcp/gpu_info.c +++ /dev/null @@ -1,55 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ -//#include -#include -#include -#include -#include - -#include "gpu_info.h" - -void compute_active_thread(size_t *thread, - size_t *grid, - int task, - int pad, - int major, - int minor, - int sm) -{ - int max_thread; - int max_block=8; - if(major==1) - { - if(minor>=2) - max_thread=1024; - else - max_thread=768; - } - else if(major==2) - max_thread=1536; - else - //newer GPU //keep using 2.0 - max_thread=1536; - - int _grid; - int _thread; - - if(task*pad>sm*max_thread) - { - _thread=max_thread/max_block; - _grid = ((task*pad+_thread-1)/_thread)*_thread; - } - else - { - _thread=pad; - _grid=task*pad; - } - - thread[0]=_thread; - grid[0]=_grid; -} diff --git a/tests/opencl/cutcp/gpu_info.h b/tests/opencl/cutcp/gpu_info.h deleted file mode 100644 index 5d433968..00000000 --- a/tests/opencl/cutcp/gpu_info.h +++ /dev/null @@ -1,28 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#ifndef __GPUINFOH__ -#define __GPUINFOH__ - -#ifdef __cplusplus -extern "C" { -#endif - -void compute_active_thread(size_t *thread, - size_t *grid, - int task, - int pad, - int major, - int minor, - int sm); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/tests/opencl/cutcp/kernel.cl b/tests/opencl/cutcp/kernel.cl deleted file mode 100644 index 7bbd24b8..00000000 --- a/tests/opencl/cutcp/kernel.cl +++ /dev/null @@ -1,104 +0,0 @@ -/* - * potential lattice is decomposed into size 8^3 lattice point "regions" - * - * THIS IMPLEMENTATION: one thread per lattice point - * thread block size 128 gives 4 thread blocks per region - * kernel is invoked for each x-y plane of regions, - * where gridDim.x is 4*(x region dimension) so that blockIdx.x - * can absorb the z sub-region index in its 2 lowest order bits - * - * Regions are stored contiguously in memory in row-major order - * - * The bins have to not only cover the region, but they need to surround - * the outer edges so that region sides and corners can still use - * neighbor list stencil. The binZeroAddr is actually a shifted pointer into - * the bin array (binZeroAddr = binBaseAddr + (c*binDim_y + c)*binDim_x + c) - * where c = ceil(cutoff / binsize). This allows for negative offsets to - * be added to myBinIndex. - * - * The (0,0,0) spatial origin corresponds to lower left corner of both - * regionZeroAddr and binZeroAddr. The atom coordinates are translated - * during binning to enforce this assumption. - */ - -#include "macros.h" - -// OpenCL 1.1 support for int3 is not uniform on all implementations, so -// we use int4 instead. Only the 'x', 'y', and 'z' fields of xyz are used. -typedef int4 xyz; - -__kernel void opencl_cutoff_potential_lattice( - int binDim_x, - int binDim_y, - __global float4 *binBaseAddr, - int offset, - float h, /* lattice spacing */ - float cutoff2, /* square of cutoff distance */ - float inv_cutoff2, - __global float *regionZeroAddr, /* address of lattice regions starting at origin */ - int zRegionIndex, - __constant int *NbrListLen, - __constant xyz *NbrList - ) -{ - __global float4* binZeroAddr = binBaseAddr + offset; - - __global float *myRegionAddr; - int Bx, By, Bz; - - /* thread id */ - const int tid = (get_local_id(2)*get_local_size(1) + - get_local_id(1))*get_local_size(0) + get_local_id(0); - - /* this is the start of the sub-region indexed by tid */ - myRegionAddr = regionZeroAddr + ((zRegionIndex*get_num_groups(1) - + get_group_id(1))*(get_num_groups(0)>>2) + (get_group_id(0)>>2))*REGION_SIZE - + (get_group_id(0)&3)*SUB_REGION_SIZE; - - /* spatial coordinate of this lattice point */ - float x = (8 * (get_group_id(0) >> 2) + get_local_id(0)) * h; - float y = (8 * get_group_id(1) + get_local_id(1)) * h; - float z = (8 * zRegionIndex + 2*(get_group_id(0)&3) + get_local_id(2)) * h; - - float dx; - float dy; - float dz; - float r2; - float s; - - int totalbins = 0; - - /* bin number determined by center of region */ - Bx = (int) floor((8 * (get_group_id(0) >> 2) + 4) * h * BIN_INVLEN); - By = (int) floor((8 * get_group_id(1) + 4) * h * BIN_INVLEN); - Bz = (int) floor((8 * zRegionIndex + 4) * h * BIN_INVLEN); - - float energy = 0.f; - int bincnt; - for (bincnt = 0; bincnt < *NbrListLen; bincnt++) { - int i = Bx + NbrList[bincnt].x; - int j = By + NbrList[bincnt].y; - int k = Bz + NbrList[bincnt].z; - - __global float4* p_global = binZeroAddr + - (((k*binDim_y + j)*binDim_x + i) * BIN_DEPTH); - - int m; - for (m = 0; m < BIN_DEPTH; m++) { - float aq = p_global[m].w; - if (0.f != aq) { - dx = p_global[m].x - x; - dy = p_global[m].y - y; - dz = p_global[m].z - z; - r2 = dx*dx + dy*dy + dz*dz; - if (r2 < cutoff2) { - s = (1.f - r2 * inv_cutoff2); - energy += aq * rsqrt(r2) * s * s; - } - } - } /* end loop over atoms in bin */ - } /* end loop over neighbor list */ - - /* store into global memory */ - myRegionAddr[tid+0] = energy; -} diff --git a/tests/opencl/cutcp/macros.h b/tests/opencl/cutcp/macros.h deleted file mode 100644 index fed9553c..00000000 --- a/tests/opencl/cutcp/macros.h +++ /dev/null @@ -1,69 +0,0 @@ -#ifndef __MACROSH__ -#define __MACROSH__ - -#ifdef __DEVICE_EMULATION__ -#define DEBUG -/* define which grid block and which thread to examine */ -#define BX 0 -#define BY 0 -#define TX 0 -#define TY 0 -#define TZ 0 -#define EMU(code) do { \ - if (blockIdx.x==BX && blockIdx.y==BY && \ - threadIdx.x==TX && threadIdx.y==TY && threadIdx.z==TZ) { \ - code; \ - } \ -} while (0) -#define INT(n) printf("%s = %d\n", #n, n) -#define FLOAT(f) printf("%s = %g\n", #f, (double)(f)) -#define INT3(n) printf("%s = %d %d %d\n", #n, (n).x, (n).y, (n).z) -#define FLOAT4(f) printf("%s = %g %g %g %g\n", #f, (double)(f).x, \ - (double)(f).y, (double)(f).z, (double)(f).w) -#else -#define EMU(code) -#define INT(n) -#define FLOAT(f) -#define INT3(n) -#define FLOAT4(f) -#endif - -/* report error from OpenCL */ -#define CHECK_ERROR(errorMessage) \ - if(clStatus != CL_SUCCESS) \ - { \ - printf("Error: %s!\n",errorMessage); \ - printf("Line: %d\n",__LINE__); \ - exit(1); \ - } - -/* - * neighbor list: - * stored in constant memory as table of offsets - * flat index addressing is computed by kernel - * - * reserve enough memory for 11^3 stencil of grid cells - * this fits within 16K of memory - */ -#define NBRLIST_DIM 11 -#define NBRLIST_MAXLEN (NBRLIST_DIM * NBRLIST_DIM * NBRLIST_DIM) - -/* - * atom bins cached into shared memory for processing - * - * this reserves 4K of shared memory for 32 atom bins each containing 8 atoms, - * should permit scheduling of up to 3 thread blocks per SM - */ -#define BIN_DEPTH 8 /* max number of atoms per bin */ -#define BIN_SIZE 32 /* size of bin in floats */ -#define BIN_CACHE_MAXLEN 32 /* max number of atom bins to cache */ - -#define BIN_LENGTH 4.f /* spatial length in Angstroms */ -#define BIN_INVLEN (1.f / BIN_LENGTH) -/* assuming density of 1 atom / 10 A^3, expectation is 6.4 atoms per bin - * so that bin fill should be 80% (for non-empty regions of space) */ - -#define REGION_SIZE 512 /* number of floats in lattice region */ -#define SUB_REGION_SIZE 128 /* number of floats in lattice sub-region */ - -#endif diff --git a/tests/opencl/cutcp/main.cc b/tests/opencl/cutcp/main.cc deleted file mode 100644 index 72ab37f3..00000000 --- a/tests/opencl/cutcp/main.cc +++ /dev/null @@ -1,190 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2008-2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#include -#include -#include -#include -#include - -#include "atom.h" -#include "cutoff.h" -#include "output.h" - -#define ERRTOL 1e-4f - -#define NOKERNELS 0 -#define CUTOFF1 1 -#define CUTOFF6 32 -#define CUTOFF6OVERLAP 64 -#define CUTOFFCPU 16384 - - -int appenddata(const char *filename, int size, double time) { - FILE *fp; - fp=fopen(filename, "a"); - if (fp == NULL) { - printf("error appending to file %s..\n", filename); - return -1; - } - fprintf(fp, "%d %.3f\n", size, time); - fclose(fp); - return 0; -} - -LatticeDim -lattice_from_bounding_box(Vec3 lo, Vec3 hi, float h) -{ - LatticeDim ret; - - ret.nx = (int) floorf((hi.x-lo.x)/h) + 1; - ret.ny = (int) floorf((hi.y-lo.y)/h) + 1; - ret.nz = (int) floorf((hi.z-lo.z)/h) + 1; - ret.lo = lo; - ret.h = h; - - return ret; -} - -Lattice * -create_lattice(LatticeDim dim) -{ - int size; - Lattice *lat = (Lattice *)malloc(sizeof(Lattice)); - - if (lat == NULL) { - fprintf(stderr, "Out of memory\n"); - exit(1); - } - - lat->dim = dim; - - /* Round up the allocated size to a multiple of 8 */ - size = ((dim.nx * dim.ny * dim.nz) + 7) & ~7; - lat->lattice = (float *)calloc(size, sizeof(float)); - - if (lat->lattice == NULL) { - fprintf(stderr, "Out of memory\n"); - exit(1); - } - - return lat; -} - - -void -destroy_lattice(Lattice *lat) -{ - if (lat) { - free(lat->lattice); - free(lat); - } -} - -int main(int argc, char *argv[]) { - Atoms *atom; - - LatticeDim lattice_dim; - Lattice *gpu_lattice; - Vec3 min_ext, max_ext; /* Bounding box of atoms */ - Vec3 lo, hi; /* Bounding box with padding */ - - float h = 0.5f; /* Lattice spacing */ - float cutoff = 12.f; /* Cutoff radius */ - float exclcutoff = 1.f; /* Radius for exclusion */ - float padding = 0.5f; /* Bounding box padding distance */ - - int n; - - struct pb_Parameters *parameters; - struct pb_TimerSet timers; - - /* Read input parameters */ - parameters = pb_ReadParameters(&argc, argv); - if (parameters == NULL) { - exit(1); - } - - parameters->inpFiles = (char **)malloc(sizeof(char *) * 2); - parameters->inpFiles[0] = (char *)malloc(100); - parameters->inpFiles[1] = NULL; - strncpy(parameters->inpFiles[0], "watbox.sl40.pqr", 100); - - /* Expect one input file */ - if (pb_Parameters_CountInputs(parameters) != 1) { - fprintf(stderr, "Expecting one input file\n"); - exit(1); - } - - pb_InitializeTimerSet(&timers); - pb_SwitchToTimer(&timers, pb_TimerID_IO); - - { - const char *pqrfilename = parameters->inpFiles[0]; - - if (!(atom = read_atom_file(pqrfilename))) { - fprintf(stderr, "read_atom_file() failed\n"); - exit(1); - } - printf("read %d atoms from file '%s'\n", atom->size, pqrfilename); - } - - /* find extent of domain */ - pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); - get_atom_extent(&min_ext, &max_ext, atom); - printf("extent of domain is:\n"); - printf(" minimum %g %g %g\n", min_ext.x, min_ext.y, min_ext.z); - printf(" maximum %g %g %g\n", max_ext.x, max_ext.y, max_ext.z); - - printf("padding domain by %g Angstroms\n", padding); - lo = (Vec3) {min_ext.x - padding, min_ext.y - padding, min_ext.z - padding}; - hi = (Vec3) {max_ext.x + padding, max_ext.y + padding, max_ext.z + padding}; - printf("domain lengths are %g by %g by %g\n", hi.x-lo.x, hi.y-lo.y, hi.z-lo.z); - - lattice_dim = lattice_from_bounding_box(lo, hi, h); - gpu_lattice = create_lattice(lattice_dim); - printf("\n"); - - /* - * Run OpenCL kernel - * (Begin and end with COMPUTE timer active) - */ - if (gpu_compute_cutoff_potential_lattice(&timers, gpu_lattice, cutoff, atom, 0, parameters)) { - fprintf(stderr, "Computation failed\n"); - exit(1); - } - - /* - * Zero the lattice points that are too close to an atom. This is - * necessary for numerical stability. - */ - if (remove_exclusions(gpu_lattice, exclcutoff, atom)) { - fprintf(stderr, "remove_exclusions() failed for gpu lattice\n"); - exit(1); - } - - printf("\n"); - - pb_SwitchToTimer(&timers, pb_TimerID_IO); - - /* Print output */ - if (parameters->outFile) { - //write_lattice_summary(parameters->outFile, gpu_lattice); - } - pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); - - /* Cleanup */ - destroy_lattice(gpu_lattice); - free_atom(atom); - - pb_SwitchToTimer(&timers, pb_TimerID_NONE); - pb_PrintTimerSet(&timers); - pb_FreeParameters(parameters); - - return 0; -} diff --git a/tests/opencl/cutcp/ocl.c b/tests/opencl/cutcp/ocl.c deleted file mode 100644 index 7e48e7e6..00000000 --- a/tests/opencl/cutcp/ocl.c +++ /dev/null @@ -1,49 +0,0 @@ -#include -#include -#include -#include "ocl.h" - -char* readFile(const char* fileName) -{ - FILE* fp; - fp = fopen(fileName,"r"); - if(fp == NULL) - { - printf("Error 1!\n"); - exit(1); - } - - fseek(fp,0,SEEK_END); - long size = ftell(fp); - rewind(fp); - - char* buffer = (char*)malloc(sizeof(char)*(size+1)); - if(buffer == NULL) - { - printf("Error 2!\n"); - fclose(fp); - exit(1); - } - - size_t res = fread(buffer,1,size,fp); - if(res != size) - { - printf("Error 3!\n"); - fclose(fp); - exit(1); - } - - buffer[size] = 0; - fclose(fp); - return buffer; -} - -void clMemSet(cl_command_queue clCommandQueue, cl_mem buf, int val, size_t size) -{ - cl_int clStatus; - char* temp = (char*)malloc(size); - memset(temp,val,size); - clStatus = clEnqueueWriteBuffer(clCommandQueue,buf,CL_TRUE,0,size,temp,0,NULL,NULL); - CHECK_ERROR("clEnqueueWriteBuffer") - free(temp); -} diff --git a/tests/opencl/cutcp/ocl.h b/tests/opencl/cutcp/ocl.h deleted file mode 100644 index 6de9a5ec..00000000 --- a/tests/opencl/cutcp/ocl.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef __OCLH__ -#define __OCLH__ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -void clMemSet(cl_command_queue, cl_mem, int, size_t); -char* readFile(const char*); - -#define CHECK_ERROR(errorMessage) \ - if(clStatus != CL_SUCCESS) \ - { \ - printf("Error: %s!\n",errorMessage); \ - printf("Line: %d\n",__LINE__); \ - exit(1); \ - } - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/tests/opencl/cutcp/output.c b/tests/opencl/cutcp/output.c deleted file mode 100644 index 36ee7e2b..00000000 --- a/tests/opencl/cutcp/output.c +++ /dev/null @@ -1,67 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2008-2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#include -#include -#include -#include -#include - -#include "atom.h" -#include "cutoff.h" - -void -write_lattice_summary(const char *filename, Lattice *lattice) -{ - float *lattice_data = lattice->lattice; - int nx = lattice->dim.nx; - int ny = lattice->dim.ny; - int nz = lattice->dim.nz; - - /* Open output file */ - FILE *outfile = fopen(filename, "w"); - - if (outfile == NULL) { - fprintf(stderr, "Cannot open output file\n"); - exit(1); - } - - /* Write the sum of the the absolute values of all lattice potentials */ - { - double abspotential = 0.0; - float tmp; - int i; - - for (i = 0; i < nx * ny * nz; i++) - abspotential += fabs((double) lattice_data[i]); - - tmp = (float) abspotential; - - fwrite(&tmp, 1, sizeof(float), outfile); - } - - /* Write the size of a lattice plane */ - { - uint32_t tmp; - - tmp = (uint32_t) (lattice->dim.nx * lattice->dim.ny); - fwrite(&tmp, 1, sizeof(uint32_t), outfile); - } - - /* Write the plane of lattice data at z=0 and z = nz-1 */ - { - int plane_size = nx * ny; - - fwrite(lattice_data, plane_size, sizeof(float), outfile); - fwrite(lattice_data + (nz-1) * plane_size, plane_size, sizeof(float), - outfile); - } - - /* Cleanup */ - fclose(outfile); -} diff --git a/tests/opencl/cutcp/output.h b/tests/opencl/cutcp/output.h deleted file mode 100644 index 13022cd9..00000000 --- a/tests/opencl/cutcp/output.h +++ /dev/null @@ -1,25 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2008-2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#ifndef OUTPUT_H -#define OUTPUT_H - -#include "cutoff.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void -write_lattice_summary(const char *filename, Lattice *lattice); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/tests/opencl/cutcp/parboil.h b/tests/opencl/cutcp/parboil.h deleted file mode 100644 index 4c9a8b5e..00000000 --- a/tests/opencl/cutcp/parboil.h +++ /dev/null @@ -1,348 +0,0 @@ -/* - * (c) 2010 The Board of Trustees of the University of Illinois. - */ -#ifndef PARBOIL_HEADER -#define PARBOIL_HEADER - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -/* A platform as specified by the user on the command line */ -struct pb_PlatformParam { - char *name; /* The platform name. This string is owned. */ - char *version; /* The platform version; may be NULL. - * This string is owned. */ -}; - -/* Create a PlatformParam from the given strings. - * 'name' must not be NULL. 'version' may be NULL. - * If not NULL, the strings should have been allocated by malloc(), - * and they will be owned by the returned object. - */ -struct pb_PlatformParam * -pb_PlatformParam(char *name, char *version); - -void -pb_FreePlatformParam(struct pb_PlatformParam *); - -/* A criterion for how to select a device */ -enum pb_DeviceSelectionCriterion { - pb_Device_INDEX, /* Enumerate the devices and select one - * by its number */ - pb_Device_CPU, /* Select a CPU device */ - pb_Device_GPU, /* Select a GPU device */ - pb_Device_ACCELERATOR, /* Select an accelerator device */ - pb_Device_NAME /* Select a device by name */ -}; - -/* A device as specified by the user on the command line */ -struct pb_DeviceParam { - enum pb_DeviceSelectionCriterion criterion; - union { - int index; /* If criterion == pb_Device_INDEX, - * the index of the device */ - char *name; /* If criterion == pb_Device_NAME, - * the name of the device. - * This string is owned. */ - }; -}; - -struct pb_DeviceParam * -pb_DeviceParam_index(int index); - -struct pb_DeviceParam * -pb_DeviceParam_cpu(void); - -struct pb_DeviceParam * -pb_DeviceParam_gpu(void); - -struct pb_DeviceParam * -pb_DeviceParam_accelerator(void); - -/* Create a by-name device selection criterion. - * The string should have been allocated by malloc(), and it will will be - * owned by the returned object. - */ -struct pb_DeviceParam * -pb_DeviceParam_name(char *name); - -void -pb_FreeDeviceParam(struct pb_DeviceParam *); - -/* Command line parameters for benchmarks */ -struct pb_Parameters { - char *outFile; /* If not NULL, the raw output of the - * computation should be saved to this - * file. The string is owned. */ - char **inpFiles; /* A NULL-terminated array of strings - * holding the input file(s) for the - * computation. The array and strings - * are owned. */ - struct pb_PlatformParam *platform; /* If not NULL, the platform - * specified on the command line. */ - struct pb_DeviceParam *device; /* If not NULL, the device - * specified on the command line. */ -}; - -/* Read command-line parameters. - * - * The argc and argv parameters to main are read, and any parameters - * interpreted by this function are removed from the argument list. - * - * A new instance of struct pb_Parameters is returned. - * If there is an error, then an error message is printed on stderr - * and NULL is returned. - */ -struct pb_Parameters * -pb_ReadParameters(int *_argc, char **argv); - -/* Free an instance of struct pb_Parameters. - */ -void -pb_FreeParameters(struct pb_Parameters *p); - -void -pb_FreeStringArray(char **); - -/* Count the number of input files in a pb_Parameters instance. - */ -int -pb_Parameters_CountInputs(struct pb_Parameters *p); - -/* A time or duration. */ -//#if _POSIX_VERSION >= 200112L -typedef unsigned long long pb_Timestamp; /* time in microseconds */ -//#else -//# error "Timestamps not implemented" -//#endif - -enum pb_TimerState { - pb_Timer_STOPPED, - pb_Timer_RUNNING, -}; - -struct pb_Timer { - enum pb_TimerState state; - pb_Timestamp elapsed; /* Amount of time elapsed so far */ - pb_Timestamp init; /* Beginning of the current time interval, - * if state is RUNNING. End of the last - * recorded time interfal otherwise. */ -}; - -/* Reset a timer. - * Use this to initialize a timer or to clear - * its elapsed time. The reset timer is stopped. - */ -void -pb_ResetTimer(struct pb_Timer *timer); - -/* Start a timer. The timer is set to RUNNING mode and - * time elapsed while the timer is running is added to - * the timer. - * The timer should not already be running. - */ -void -pb_StartTimer(struct pb_Timer *timer); - -/* Stop a timer. - * This stops adding elapsed time to the timer. - * The timer should not already be stopped. - */ -void -pb_StopTimer(struct pb_Timer *timer); - -/* Get the elapsed time in seconds. */ -double -pb_GetElapsedTime(struct pb_Timer *timer); - -/* Execution time is assigned to one of these categories. */ -enum pb_TimerID { - pb_TimerID_NONE = 0, - pb_TimerID_IO, /* Time spent in input/output */ - pb_TimerID_KERNEL, /* Time spent computing on the device, - * recorded asynchronously */ - pb_TimerID_COPY, /* Time spent synchronously moving data - * to/from device and allocating/freeing - * memory on the device */ - pb_TimerID_DRIVER, /* Time spent in the host interacting with the - * driver, primarily for recording the time - * spent queueing asynchronous operations */ - pb_TimerID_COPY_ASYNC, /* Time spent in asynchronous transfers */ - pb_TimerID_COMPUTE, /* Time for all program execution other - * than parsing command line arguments, - * I/O, kernel, and copy */ - pb_TimerID_OVERLAP, /* Time double-counted in asynchronous and - * host activity: automatically filled in, - * not intended for direct usage */ - pb_TimerID_LAST /* Number of timer IDs */ -}; - -/* Dynamic list of asynchronously tracked times between events */ -struct pb_async_time_marker_list { - char *label; // actually just a pointer to a string - enum pb_TimerID timerID; /* The ID to which the interval beginning - * with this marker should be attributed */ - void * marker; - //cudaEvent_t marker; /* The driver event for this marker */ - struct pb_async_time_marker_list *next; -}; - -struct pb_SubTimer { - char *label; - struct pb_Timer timer; - struct pb_SubTimer *next; -}; - -struct pb_SubTimerList { - struct pb_SubTimer *current; - struct pb_SubTimer *subtimer_list; -}; - -/* A set of timers for recording execution times. */ -struct pb_TimerSet { - enum pb_TimerID current; - struct pb_async_time_marker_list* async_markers; - pb_Timestamp async_begin; - pb_Timestamp wall_begin; - struct pb_Timer timers[pb_TimerID_LAST]; - struct pb_SubTimerList *sub_timer_list[pb_TimerID_LAST]; -}; - -/* Reset all timers in the set. */ -void -pb_InitializeTimerSet(struct pb_TimerSet *timers); - -void -pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category); - -/* Select which timer the next interval of time should be accounted - * to. The selected timer is started and other timers are stopped. - * Using pb_TimerID_NONE stops all timers. */ -void -pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer); - -void -pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID category); - -/* Print timer values to standard output. */ -void -pb_PrintTimerSet(struct pb_TimerSet *timers); - -/* Release timer resources */ -void -pb_DestroyTimerSet(struct pb_TimerSet * timers); - -void -pb_SetOpenCL(void *clContextPtr, void *clCommandQueuePtr); - - -typedef struct pb_Device_tag { - char* name; - void* clDevice; - int id; - unsigned int in_use; - unsigned int available; -} pb_Device; - -struct pb_Context_tag; -typedef struct pb_Context_tag pb_Context; - -typedef struct pb_Platform_tag { - char* name; - char* version; - void* clPlatform; - unsigned int in_use; - pb_Context** contexts; - pb_Device** devices; -} pb_Platform; - -struct pb_Context_tag { - void* clPlatformId; - void* clContext; - void* clDeviceId; - pb_Platform* pb_platform; - pb_Device* pb_device; -}; - -// verbosely print out list of platforms and their devices to the console. -pb_Platform** -pb_GetPlatforms(); - -// Choose a platform according to the given platform specification -pb_Platform* -pb_GetPlatform(struct pb_PlatformParam *platform); - -// choose a platform: by name, name & version -pb_Platform* -pb_GetPlatformByName(const char* name); - -pb_Platform* -pb_GetPlatformByNameAndVersion(const char* name, const char* version); - -// Choose a device according to the given device specification -pb_Device* -pb_GetDevice(pb_Platform* pb_platform, struct pb_DeviceParam *device); - -pb_Device** -pb_GetDevices(pb_Platform* pb_platform); - -// choose a device by name. -pb_Device* -pb_GetDeviceByName(pb_Platform* pb_platform, const char* name); - -pb_Platform* -pb_GetPlatformByEnvVars(); - -pb_Context* -pb_InitOpenCLContext(struct pb_Parameters* parameters); - -void -pb_ReleasePlatforms(); - -void -pb_ReleaseContext(pb_Context* c); - -void -pb_PrintPlatformInfo(pb_Context* c); - -void -perf_init(); - -//#define MEASURE_KERNEL_TIME - -#include - -#ifdef MEASURE_KERNEL_TIME -#define clEnqueueNDRangeKernel(q,k,d,o,dg,db,a,b,c) pb_clEnqueueNDRangeKernel((q), (k), (d), (o), (dg), (db), (a), (b), (c)) -cl_int -pb_clEnqueueNDRangeKernel(cl_command_queue /* command_queue */, - cl_kernel /* kernel */, - cl_uint /* work_dim */, - const size_t * /* global_work_offset */, - const size_t * /* global_work_size */, - const size_t * /* local_work_size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */); -#endif - -enum { T_FLOAT, T_DOUBLE, T_SHORT, T_INT, T_UCHAR }; -void pb_sig_float(char*, float*, int); -void pb_sig_double(char*, double*, int); -void pb_sig_short(char*, short*, int); -void pb_sig_int(char*, int*, int); -void pb_sig_uchar(char*, unsigned char*, unsigned int); -void pb_sig_clmem(char*, cl_command_queue, cl_mem, int); - -#ifdef __cplusplus -} -#endif - -#endif //PARBOIL_HEADER - diff --git a/tests/opencl/cutcp/parboil_opencl.c b/tests/opencl/cutcp/parboil_opencl.c deleted file mode 100644 index a4db1680..00000000 --- a/tests/opencl/cutcp/parboil_opencl.c +++ /dev/null @@ -1,1394 +0,0 @@ -/* - * (c) 2007 The Board of Trustees of the University of Illinois. - */ - -#include -#include -#include -#include -#include -#include - -#if _POSIX_VERSION >= 200112L -# include -#endif - -//#include "perfmon.h" - -cl_context *clContextPtr; -cl_command_queue *clCommandQueuePtr; - -// #define DISABLE_PARBOIL_TIMER - -/*****************************************************************************/ -/* Timer routines */ - -static int is_async(enum pb_TimerID timer) -{ - return (timer == pb_TimerID_KERNEL) || - (timer == pb_TimerID_COPY_ASYNC); -} - -static int is_blocking(enum pb_TimerID timer) -{ - return (timer == pb_TimerID_COPY) || (timer == pb_TimerID_NONE); -} - -#define INVALID_TIMERID pb_TimerID_LAST - -static int asyncs_outstanding(struct pb_TimerSet* timers) -{ - return (timers->async_markers != NULL) && - (timers->async_markers->timerID != INVALID_TIMERID); -} - -static struct pb_async_time_marker_list * -get_last_async(struct pb_TimerSet* timers) -{ - /* Find the last event recorded thus far */ - struct pb_async_time_marker_list * last_event = timers->async_markers; - if(last_event != NULL && last_event->timerID != INVALID_TIMERID) { - while(last_event->next != NULL && - last_event->next->timerID != INVALID_TIMERID) - last_event = last_event->next; - return last_event; - } else - return NULL; -} - -static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* - // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating Marker [%d]\n", timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = NULL; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - -static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = label; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - - -/* Assumes that all recorded events have completed */ -static pb_Timestamp record_async_times(struct pb_TimerSet* tset) -{ - struct pb_async_time_marker_list * next_interval = NULL; - struct pb_async_time_marker_list * last_marker = get_last_async(tset); - pb_Timestamp total_async_time = 0; - enum pb_TimerID timer; - - for(next_interval = tset->async_markers; next_interval != last_marker; - next_interval = next_interval->next) { - cl_ulong command_start=0, command_end=0; - cl_int ciErrNum = CL_SUCCESS; - - ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_start, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error getting first EventProfilingInfo: %d\n", ciErrNum); - } - - ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->next->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_end, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error getting second EventProfilingInfo: %d\n", ciErrNum); - } - - pb_Timestamp interval = (pb_Timestamp) (((double)(command_end - command_start)) / 1e3); - tset->timers[next_interval->timerID].elapsed += interval; - if (next_interval->label != NULL) { - struct pb_SubTimer *subtimer = tset->sub_timer_list[next_interval->timerID]->subtimer_list; - while (subtimer != NULL) { - if ( strcmp(subtimer->label, next_interval->label) == 0) { - subtimer->timer.elapsed += interval; - break; - } - subtimer = subtimer->next; - } - } - total_async_time += interval; - next_interval->timerID = INVALID_TIMERID; - } - - if(next_interval != NULL) - next_interval->timerID = INVALID_TIMERID; - - return total_async_time; -} - -static void -accumulate_time(pb_Timestamp *accum, - pb_Timestamp start, - pb_Timestamp end) -{ -//#if _POSIX_VERSION >= 200112L - *accum += end - start; -//#else -//# error "Timestamps not implemented for this system" -//#endif -} - -//#if _POSIX_VERSION >= 200112L -static pb_Timestamp get_time() -{ - //struct timeval tv; - //gettimeofday(&tv, NULL); - //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); - return 0; -} -//#else -//# error "no supported time libraries are available on this platform" -//#endif - -void -pb_ResetTimer(struct pb_Timer *timer) -{ -//#ifndef DISABLE_PARBOIL_TIMER - timer->state = pb_Timer_STOPPED; - -//#if _POSIX_VERSION >= 200112L - timer->elapsed = 0; -//#else -//# error "pb_ResetTimer: not implemented for this system" -//#endif -//#endif -} - -void -pb_StartTimer(struct pb_Timer *timer) -{ -/*#ifndef DISABLE_PARBOIL_TIMER - if (timer->state != pb_Timer_STOPPED) { - fputs("Ignoring attempt to start a running timer\n", stderr); - return; - } - - timer->state = pb_Timer_RUNNING; - -#if _POSIX_VERSION >= 200112L - { - struct timeval tv; - gettimeofday(&tv, NULL); - timer->init = tv.tv_sec * 1000000LL + tv.tv_usec; - } -#else -# error "pb_StartTimer: not implemented for this system" -#endif -#endif*/ -} - -void -pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) -{ -/*#ifndef DISABLE_PARBOIL_TIMER - - unsigned int numNotStopped = 0x3; // 11 - if (timer->state != pb_Timer_STOPPED) { - fputs("Warning: Timer was not stopped\n", stderr); - numNotStopped &= 0x1; // Zero out 2^1 - } - if (subtimer->state != pb_Timer_STOPPED) { - fputs("Warning: Subtimer was not stopped\n", stderr); - numNotStopped &= 0x2; // Zero out 2^0 - } - if (numNotStopped == 0x0) { - fputs("Ignoring attempt to start running timer and subtimer\n", stderr); - return; - } - - timer->state = pb_Timer_RUNNING; - subtimer->state = pb_Timer_RUNNING; - -#if _POSIX_VERSION >= 200112L - { - struct timeval tv; - gettimeofday(&tv, NULL); - - if (numNotStopped & 0x2) { - timer->init = tv.tv_sec * 1000000LL + tv.tv_usec; - } - - if (numNotStopped & 0x1) { - subtimer->init = tv.tv_sec * 1000000LL + tv.tv_usec; - } - } -#else -# error "pb_StartTimer: not implemented for this system" -#endif - -#endif*/ -} - -void -pb_StopTimer(struct pb_Timer *timer) -{ -/*#ifndef DISABLE_PARBOIL_TIMER - - pb_Timestamp fini; - - if (timer->state != pb_Timer_RUNNING) { - fputs("Ignoring attempt to stop a stopped timer\n", stderr); - return; - } - - timer->state = pb_Timer_STOPPED; - -#if _POSIX_VERSION >= 200112L - { - struct timeval tv; - gettimeofday(&tv, NULL); - fini = tv.tv_sec * 1000000LL + tv.tv_usec; - } -#else -# error "pb_StopTimer: not implemented for this system" -#endif - - accumulate_time(&timer->elapsed, timer->init, fini); - timer->init = fini; - -#endif*/ -} - -void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { -/*#ifndef DISABLE_PARBOIL_TIMER - - pb_Timestamp fini; - - unsigned int numNotRunning = 0x3; // 11 - if (timer->state != pb_Timer_RUNNING) { - fputs("Warning: Timer was not running\n", stderr); - numNotRunning &= 0x1; // Zero out 2^1 - } - if (subtimer->state != pb_Timer_RUNNING) { - fputs("Warning: Subtimer was not running\n", stderr); - numNotRunning &= 0x2; // Zero out 2^0 - } - if (numNotRunning == 0x0) { - fputs("Ignoring attempt to stop stopped timer and subtimer\n", stderr); - return; - } - - - timer->state = pb_Timer_STOPPED; - subtimer->state = pb_Timer_STOPPED; - -#if _POSIX_VERSION >= 200112L - { - struct timeval tv; - gettimeofday(&tv, NULL); - fini = tv.tv_sec * 1000000LL + tv.tv_usec; - } -#else -# error "pb_StopTimer: not implemented for this system" -#endif - - if (numNotRunning & 0x2) { - accumulate_time(&timer->elapsed, timer->init, fini); - timer->init = fini; - } - - if (numNotRunning & 0x1) { - accumulate_time(&subtimer->elapsed, subtimer->init, fini); - subtimer->init = fini; - } - -#endif*/ -} - -/* Get the elapsed time in seconds. */ -double -pb_GetElapsedTime(struct pb_Timer *timer) -{ - /*double ret; -#ifndef DISABLE_PARBOIL_TIMER - - if (timer->state != pb_Timer_STOPPED) { - fputs("Elapsed time from a running timer is inaccurate\n", stderr); - } - -#if _POSIX_VERSION >= 200112L - ret = timer->elapsed / 1e6; -#else -# error "pb_GetElapsedTime: not implemented for this system" -#endif -#endif - return ret;*/ - return 0; -} - -void -pb_InitializeTimerSet(struct pb_TimerSet *timers) -{ -/*#ifndef DISABLE_PARBOIL_TIMER - int n; - - timers->wall_begin = 0; //get_time(); - timers->current = pb_TimerID_NONE; - - timers->async_markers = NULL; - - for (n = 0; n < pb_TimerID_LAST; n++) { - pb_ResetTimer(&timers->timers[n]); - timers->sub_timer_list[n] = NULL; - } -#endif*/ -} - -void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { - clContextPtr = ((cl_context *)p_clContextPtr); - clCommandQueuePtr = ((cl_command_queue *)p_clCommandQueuePtr); -} - -void -pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { -/*#ifndef DISABLE_PARBOIL_TIMER - - struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc - (sizeof(struct pb_SubTimer)); - - int len = strlen(label); - - subtimer->label = (char *) malloc (sizeof(char)*(len+1)); - sprintf(subtimer->label, "%s\0", label); - - pb_ResetTimer(&subtimer->timer); - subtimer->next = NULL; - - struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[pb_Category]; - if (subtimerlist == NULL) { - subtimerlist = (struct pb_SubTimerList *) calloc - (1, sizeof(struct pb_SubTimerList)); - subtimerlist->subtimer_list = subtimer; - timers->sub_timer_list[pb_Category] = subtimerlist; - } else { - // Append to list - struct pb_SubTimer *element = subtimerlist->subtimer_list; - while (element->next != NULL) { - element = element->next; - } - element->next = subtimer; - } - -#endif*/ -} - -void -pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) -{ -#if 0 -#ifndef DISABLE_PARBOIL_TIMER - - /* Stop the currently running timer */ - if (timers->current != pb_TimerID_NONE) { - struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[timers->current]; - struct pb_SubTimer *currSubTimer = (subtimerlist != NULL) ? subtimerlist->current : NULL; - - if (!is_async(timers->current) ) { - if (timers->current != timer) { - if (currSubTimer != NULL) { - pb_StopTimerAndSubTimer(&timers->timers[timers->current], &currSubTimer->timer); - } else { - pb_StopTimer(&timers->timers[timers->current]); - } - } else { - if (currSubTimer != NULL) { - pb_StopTimer(&currSubTimer->timer); - } - } - } else { - insert_marker(timers, timer); - if (!is_async(timer)) { // if switching to async too, keep driver going - pb_StopTimer(&timers->timers[pb_TimerID_DRIVER]); - } - } - } - - pb_Timestamp currentTime = 0; //get_time(); - - /* The only cases we check for asynchronous task completion is - * when an overlapping CPU operation completes, or the next - * segment blocks on completion of previous async operations */ - if( asyncs_outstanding(timers) && - (!is_async(timers->current) || is_blocking(timer) ) ) { - - struct pb_async_time_marker_list * last_event = get_last_async(timers); - /* CL_COMPLETE if completed */ - - cl_int ciErrNum = CL_SUCCESS; - cl_int async_done = CL_COMPLETE; - - ciErrNum = clGetEventInfo(*((cl_event *)last_event->marker), CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof(cl_int), &async_done, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Querying EventInfo!\n"); - } - - - if(is_blocking(timer)) { - /* Async operations completed after previous CPU operations: - * overlapped time is the total CPU time since this set of async - * operations were first issued */ - - // timer to switch to is COPY or NONE - if(async_done != CL_COMPLETE) { - accumulate_time(&(timers->timers[pb_TimerID_OVERLAP].elapsed), - timers->async_begin,currentTime); - } - - /* Wait on async operation completion */ - ciErrNum = clWaitForEvents(1, (cl_event *)last_event->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Waiting for Events!\n"); - } - - pb_Timestamp total_async_time = record_async_times(timers); - - /* Async operations completed before previous CPU operations: - * overlapped time is the total async time */ - if(async_done == CL_COMPLETE) { - //fprintf(stderr, "Async_done: total_async_type = %lld\n", total_async_time); - timers->timers[pb_TimerID_OVERLAP].elapsed += total_async_time; - } - - } else - /* implies (!is_async(timers->current) && asyncs_outstanding(timers)) */ - // i.e. Current Not Async (not KERNEL/COPY_ASYNC) but there are outstanding - // so something is deeper in stack - if(async_done == CL_COMPLETE ) { - /* Async operations completed before previous CPU operations: - * overlapped time is the total async time */ - timers->timers[pb_TimerID_OVERLAP].elapsed += record_async_times(timers); - } - } - - /* Start the new timer */ - if (timer != pb_TimerID_NONE) { - if(!is_async(timer)) { - pb_StartTimer(&timers->timers[timer]); - } else { - // toSwitchTo Is Async (KERNEL/COPY_ASYNC) - if (!asyncs_outstanding(timers)) { - /* No asyncs outstanding, insert a fresh async marker */ - - insert_marker(timers, timer); - timers->async_begin = currentTime; - } else if(!is_async(timers->current)) { - /* Previous asyncs still in flight, but a previous SwitchTo - * already marked the end of the most recent async operation, - * so we can rename that marker as the beginning of this async - * operation */ - - struct pb_async_time_marker_list * last_event = get_last_async(timers); - last_event->label = NULL; - last_event->timerID = timer; - } - if (!is_async(timers->current)) { - pb_StartTimer(&timers->timers[pb_TimerID_DRIVER]); - } - } - } - timers->current = timer; - -#endif -#endif -} - -void -pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID category) -{ -#if 0 -#ifndef DISABLE_PARBOIL_TIMER - struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[timers->current]; - struct pb_SubTimer *curr = (subtimerlist != NULL) ? subtimerlist->current : NULL; - - if (timers->current != pb_TimerID_NONE) { - if (!is_async(timers->current) ) { - if (timers->current != category) { - if (curr != NULL) { - pb_StopTimerAndSubTimer(&timers->timers[timers->current], &curr->timer); - } else { - pb_StopTimer(&timers->timers[timers->current]); - } - } else { - if (curr != NULL) { - pb_StopTimer(&curr->timer); - } - } - } else { - insert_submarker(timers, label, category); - if (!is_async(category)) { // if switching to async too, keep driver going - pb_StopTimer(&timers->timers[pb_TimerID_DRIVER]); - } - } - } - - pb_Timestamp currentTime = 0; //get_time(); - - /* The only cases we check for asynchronous task completion is - * when an overlapping CPU operation completes, or the next - * segment blocks on completion of previous async operations */ - if( asyncs_outstanding(timers) && - (!is_async(timers->current) || is_blocking(category) ) ) { - - struct pb_async_time_marker_list * last_event = get_last_async(timers); - /* CL_COMPLETE if completed */ - - cl_int ciErrNum = CL_SUCCESS; - cl_int async_done = CL_COMPLETE; - - ciErrNum = clGetEventInfo(*((cl_event *)last_event->marker), CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof(cl_int), &async_done, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Querying EventInfo!\n"); - } - - if(is_blocking(category)) { - /* Async operations completed after previous CPU operations: - * overlapped time is the total CPU time since this set of async - * operations were first issued */ - - // timer to switch to is COPY or NONE - // if it hasn't already finished, then just take now and use that as the elapsed time in OVERLAP - // anything happening after now isn't OVERLAP because everything is being stopped to wait for synchronization - // it seems that the extra sync wall time isn't being recorded anywhere - if(async_done != CL_COMPLETE) - accumulate_time(&(timers->timers[pb_TimerID_OVERLAP].elapsed), - timers->async_begin,currentTime); - - /* Wait on async operation completion */ - ciErrNum = clWaitForEvents(1, (cl_event *)last_event->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Waiting for Events!\n"); - } - pb_Timestamp total_async_time = record_async_times(timers); - - /* Async operations completed before previous CPU operations: - * overlapped time is the total async time */ - // If it did finish, then accumulate all the async time that did happen into OVERLAP - // the immediately preceding EventSynchronize theoretically didn't have any effect since it was already completed. - if(async_done == CL_COMPLETE /*cudaSuccess*/) - timers->timers[pb_TimerID_OVERLAP].elapsed += total_async_time; - - } else - /* implies (!is_async(timers->current) && asyncs_outstanding(timers)) */ - // i.e. Current Not Async (not KERNEL/COPY_ASYNC) but there are outstanding - // so something is deeper in stack - if(async_done == CL_COMPLETE /*cudaSuccess*/) { - /* Async operations completed before previous CPU operations: - * overlapped time is the total async time */ - timers->timers[pb_TimerID_OVERLAP].elapsed += record_async_times(timers); - } - // else, this isn't blocking, so just check the next time around - } - - subtimerlist = timers->sub_timer_list[category]; - struct pb_SubTimer *subtimer = NULL; - - if (label != NULL) { - subtimer = subtimerlist->subtimer_list; - while (subtimer != NULL) { - if (strcmp(subtimer->label, label) == 0) { - break; - } else { - subtimer = subtimer->next; - } - } - } - - /* Start the new timer */ - if (category != pb_TimerID_NONE) { - if(!is_async(category)) { - if (subtimerlist != NULL) { - subtimerlist->current = subtimer; - } - - if (category != timers->current && subtimer != NULL) { - pb_StartTimerAndSubTimer(&timers->timers[category], &subtimer->timer); - } else if (subtimer != NULL) { - pb_StartTimer(&subtimer->timer); - } else { - pb_StartTimer(&timers->timers[category]); - } - } else { - if (subtimerlist != NULL) { - subtimerlist->current = subtimer; - } - - // toSwitchTo Is Async (KERNEL/COPY_ASYNC) - if (!asyncs_outstanding(timers)) { - /* No asyncs outstanding, insert a fresh async marker */ - insert_submarker(timers, label, category); - timers->async_begin = currentTime; - } else if(!is_async(timers->current)) { - /* Previous asyncs still in flight, but a previous SwitchTo - * already marked the end of the most recent async operation, - * so we can rename that marker as the beginning of this async - * operation */ - - struct pb_async_time_marker_list * last_event = get_last_async(timers); - last_event->timerID = category; - last_event->label = label; - } // else, marker for switchToThis was already inserted - - //toSwitchto is already asynchronous, but if current/prev state is async too, then DRIVER is already running - if (!is_async(timers->current)) { - pb_StartTimer(&timers->timers[pb_TimerID_DRIVER]); - } - } - } - - timers->current = category; -#endif -#endif -} - -void -pb_PrintTimerSet(struct pb_TimerSet *timers) -{ -#if 0 -#ifndef DISABLE_PARBOIL_TIMER - pb_Timestamp wall_end = 0; //get_time(); - - struct pb_Timer *t = timers->timers; - struct pb_SubTimer* sub = NULL; - - int maxSubLength; - - const char *categories[] = { - "IO", "Kernel", "Copy", "Driver", "Copy Async", "Compute" - }; - - const int maxCategoryLength = 10; - - int i; - for(i = 1; i < pb_TimerID_LAST-1; ++i) { // exclude NONE and OVRELAP from this format - if(pb_GetElapsedTime(&t[i]) != 0) { - - // Print Category Timer - printf("%-*s: %f\n", maxCategoryLength, categories[i-1], pb_GetElapsedTime(&t[i])); - - if (timers->sub_timer_list[i] != NULL) { - sub = timers->sub_timer_list[i]->subtimer_list; - maxSubLength = 0; - while (sub != NULL) { - // Find longest SubTimer label - if (strlen(sub->label) > maxSubLength) { - maxSubLength = strlen(sub->label); - } - sub = sub->next; - } - - // Fit to Categories - if (maxSubLength <= maxCategoryLength) { - maxSubLength = maxCategoryLength; - } - - sub = timers->sub_timer_list[i]->subtimer_list; - - // Print SubTimers - while (sub != NULL) { - printf(" -%-*s: %f\n", maxSubLength, sub->label, pb_GetElapsedTime(&sub->timer)); - sub = sub->next; - } - } - } - } - - if(pb_GetElapsedTime(&t[pb_TimerID_OVERLAP]) != 0) - printf("CPU/Kernel Overlap: %f\n", pb_GetElapsedTime(&t[pb_TimerID_OVERLAP])); - - float walltime = (wall_end - timers->wall_begin)/ 1e6; - printf("Timer Wall Time: %f\n", walltime); - -#endif -#endif -} - -void pb_DestroyTimerSet(struct pb_TimerSet * timers) -{ -#ifndef DISABLE_PARBOIL_TIMER - /* clean up all of the async event markers */ - struct pb_async_time_marker_list* event = timers->async_markers; - while(event != NULL) { - - cl_int ciErrNum = CL_SUCCESS; - ciErrNum = clWaitForEvents(1, (cl_event *)(event)->marker); - if (ciErrNum != CL_SUCCESS) { - //fprintf(stderr, "Error Waiting for Events!\n"); - } - - ciErrNum = clReleaseEvent( *((cl_event *)(event)->marker) ); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Release Events!\n"); - } - - free((event)->marker); - struct pb_async_time_marker_list* next = ((event)->next); - - free(event); - - // (*event) = NULL; - event = next; - } - - int i = 0; - for(i = 0; i < pb_TimerID_LAST; ++i) { - if (timers->sub_timer_list[i] != NULL) { - struct pb_SubTimer *subtimer = timers->sub_timer_list[i]->subtimer_list; - struct pb_SubTimer *prev = NULL; - while (subtimer != NULL) { - free(subtimer->label); - prev = subtimer; - subtimer = subtimer->next; - free(prev); - } - free(timers->sub_timer_list[i]); - } - } -#endif -} - -static pb_Platform** ptr = NULL; - -// verbosely print out list of platforms and their devices to the console. -pb_Platform** -pb_GetPlatforms() { - if (ptr == NULL) { - cl_uint num_platforms; - clGetPlatformIDs(0, NULL, &num_platforms); - if (num_platforms == 0) return NULL; - - ptr = (pb_Platform **) malloc(sizeof(pb_Platform *) * (num_platforms + 1)); - cl_platform_id* ids = (cl_platform_id *) malloc(num_platforms * sizeof(cl_platform_id)); - clGetPlatformIDs(num_platforms, ids, NULL); - - unsigned int i; - for (i = 0; i < num_platforms; i++) { - ptr[i] = (pb_Platform *) malloc(sizeof(pb_Platform)); - ptr[i]->clPlatform = ids[i]; - ptr[i]->contexts = NULL; - ptr[i]->in_use = 0; - ptr[i]->devices = NULL; - - size_t sz; - clGetPlatformInfo(ids[i], CL_PLATFORM_NAME, 0, NULL, &sz); - char* name = (char *) malloc(sz + 1); - clGetPlatformInfo(ids[i], CL_PLATFORM_NAME, sz, name, NULL); - name[sz] = '\0'; - ptr[i]->name = name; - - clGetPlatformInfo(ids[i], CL_PLATFORM_VERSION, 0, NULL, &sz); - char* version = (char *) malloc(sz + 1); - clGetPlatformInfo(ids[i], CL_PLATFORM_VERSION, sz, version, NULL); - version[sz] = '\0'; - ptr[i]->version = version; - } - ptr[i] = NULL; - - free(ids); - } - - return (pb_Platform**) ptr; -} - -pb_Context* -createContext(pb_Platform* pb_platform, pb_Device* pb_device) { - pb_Context* c = (pb_Context*) malloc(sizeof(pb_Context)); - cl_int clStatus; - cl_context_properties clCps[3] = { - CL_CONTEXT_PLATFORM, (cl_context_properties)(pb_platform->clPlatform), 0 - }; - c->clContext = - clCreateContext(clCps, 1, (cl_device_id*)&pb_device->clDevice, NULL, NULL, &clStatus); - c->clPlatformId = pb_platform->clPlatform; - c->clDeviceId = pb_device->clDevice; - c->pb_platform = pb_platform; - c->pb_device = pb_device; - pb_platform->in_use = 1; - pb_device->in_use = 1; - unsigned int i = 0; - if (pb_platform->contexts == NULL) { - pb_platform->contexts = (pb_Context**) malloc(2*sizeof(pb_Context*)); - } else { - for (i = 0; pb_platform->contexts[i] != NULL; i++) {}; - pb_platform->contexts = (pb_Context**) realloc(pb_platform->contexts, - (i+1)*sizeof(pb_Context*)); - } - pb_platform->contexts[i+1] = NULL; - pb_platform->contexts[i] = c; - return c; -} - -// choose a platform by name. -pb_Platform* -pb_GetPlatformByName(const char* name) { - pb_Platform** ps = (pb_Platform **) pb_GetPlatforms(); - if (ps == NULL) return NULL; - if (name == NULL) { - return *ps; - } - - while (*ps) { - if (strstr((*ps)->name, name)) break; - ps++; - } - return (pb_Platform*) *ps; -} - -pb_Device** -pb_GetDevices(pb_Platform* pb_platform) { - if (pb_platform->devices == NULL) { - cl_uint num_devs; - cl_device_id* dev_ids; - clGetDeviceIDs((cl_platform_id) pb_platform->clPlatform, - CL_DEVICE_TYPE_ALL, 0, NULL, &num_devs); - if (num_devs == 0) return NULL; - - pb_platform->devices = - (pb_Device **) malloc((num_devs + 1) * sizeof(pb_Device *)); - dev_ids = (cl_device_id *) malloc(sizeof(cl_device_id) * num_devs); - clGetDeviceIDs((cl_platform_id) pb_platform->clPlatform, - CL_DEVICE_TYPE_ALL, num_devs, dev_ids, NULL); - - unsigned int i; - for (i = 0; i < num_devs; i++) { - pb_platform->devices[i] = (pb_Device *) malloc(sizeof(pb_Device)); - - pb_platform->devices[i]->clDevice = dev_ids[i]; - pb_platform->devices[i]->id = i; - - size_t sz; - clGetDeviceInfo(dev_ids[i], CL_DEVICE_NAME, 0, NULL, &sz); - char* name = (char *) malloc(sz + 1); - clGetDeviceInfo(dev_ids[i], CL_DEVICE_NAME, sz, name, NULL); - name[sz] = '\0'; - pb_platform->devices[i]->name = (char *) name; - - cl_bool available; - clGetDeviceInfo(dev_ids[i], CL_DEVICE_AVAILABLE, sizeof(cl_bool), &available, NULL); - pb_platform->devices[i]->available = (int) available; - - pb_platform->devices[i]->in_use = 0; - } - pb_platform->devices[i] = NULL; - } - return (pb_Device **) pb_platform->devices; -} - -// choose a device by name. -static pb_Device* -pb_SelectDeviceByName(pb_Device **ds, const char* name) { - if (ds == NULL) return NULL; - if (name == NULL) return *ds; - while (*ds) { - if (strstr((*ds)->name, name)) break; - ds++; - } - - return *ds; -} - -// choose a device by name and set the device's 'in_use' flag. -pb_Device* -pb_GetDeviceByName(pb_Platform* pb_platform, const char* name) { - pb_Device** ds = (pb_Device **) pb_GetDevices(pb_platform); - pb_Device *d = pb_SelectDeviceByName(ds, name); - - if (d) d->in_use = 1; - - return d; -} - -void -pb_ReleasePlatforms() { - if (!ptr) return; - pb_Platform** cur_ptr = ptr; - while (*cur_ptr) { - pb_Platform* pfptr = *cur_ptr++; - if (pfptr->devices) { - pb_Device** dvptr = pfptr->devices; - while (*dvptr) { - pb_Device* d = *dvptr++; - free(d->name); - free(d); - } - free(pfptr->devices); - } - if (pfptr->contexts) { - pb_Context** cptr = pfptr->contexts; - while (*cptr) { - free(*cptr++); - } - free(pfptr->contexts); - } - free(pfptr->name); - free(pfptr); - } - free(ptr); - ptr = NULL; -} - -pb_Platform* -pb_GetPlatformByNameAndVersion(const char* name, const char* version) { - pb_Platform** ps = (pb_Platform **) pb_GetPlatforms(); - if (ps == NULL) return NULL; - if (name == NULL) return *ps; - while (*ps) { - if (strstr((*ps)->name, name) && strstr((*ps)->version, version)) break; - ps++; - } - return (pb_Platform*) *ps; -} - -/* Return a pointer to the device at the specified index, or NULL. - * Used by pb_GetDevice. */ -static pb_Device * -select_device_by_index(pb_Device** ds, int id) -{ - int i = 0; - pb_Device** p = ds; - while (*p && (i < id)) { p++; i++; } - return *p; -} - -/* Return a pointer to the device with the specified type, or NULL. - * Used by pb_GetDevice. */ -static pb_Device * -select_device_by_type(pb_Device** ds, - enum pb_DeviceSelectionCriterion criterion) -{ - cl_device_type sought_type; - - /* Determine the OpenCL device type to search for */ - switch(criterion) { - case pb_Device_CPU: - sought_type = CL_DEVICE_TYPE_CPU; - break; - case pb_Device_GPU: - sought_type = CL_DEVICE_TYPE_GPU; - break; - case pb_Device_ACCELERATOR: - sought_type = CL_DEVICE_TYPE_ACCELERATOR; - break; - default: - fprintf(stderr, "pb_GetDevice: Invalid device type"); - exit(-1); - } - - /* Find the device */ - { - pb_Device** p = ds; - cl_device_type type; - while (*p) { - clGetDeviceInfo(((cl_device_id) ((*p)->clDevice)), CL_DEVICE_TYPE, - sizeof(cl_device_type), &type, NULL); - if (type == sought_type) break; - } - - return *p; - } -} - -pb_Device* -pb_GetDevice(pb_Platform* pb_platform, struct pb_DeviceParam *device) -{ - pb_Device** ds = (pb_Device **) pb_GetDevices(pb_platform); - - // The list of devices must be nonempty - if (ds == NULL || *ds == NULL) { - fprintf(stderr, "Error: No device is found in platform: name = %s, version = %s\n.", pb_platform->name, pb_platform->version); - exit(-1); - } - - pb_Device *selected_device = NULL; - - if (device != NULL) { - /* Use 'device' to select and return a device. - * If unable to select a device, fall - * back on the default selection mechanism. */ - switch(device->criterion) { - case pb_Device_INDEX: - selected_device = select_device_by_index(ds, device->index); - break; - case pb_Device_GPU: - case pb_Device_CPU: - case pb_Device_ACCELERATOR: - selected_device = select_device_by_type(ds, device->criterion); - break; - case pb_Device_NAME: - selected_device = pb_SelectDeviceByName(ds, device->name); - break; - default: - fprintf(stderr, "pb_GetDevice: Invalid argument"); - exit(-1); - } - } - - /* By default or if user-specified selection failed, - * select the first device */ - if (selected_device == NULL) - selected_device = *ds; - - /* Set the in_use flag */ - selected_device->in_use = 1; - - return selected_device; -} - -pb_Device* -pb_GetDeviceByEnvVars(pb_Platform* pb_platform) { - - /* Convert environment variables to a 'pb_DeviceParam' */ - struct pb_DeviceParam *param = NULL; - - char* device_num = getenv("PARBOIL_DEVICE_NUMBER"); - if (device_num && strcmp(device_num, "")) { - int id = atoi(device_num); - param = pb_DeviceParam_index(id); - } - else { - char* device_name = getenv("PARBOIL_DEVICE_NAME"); - if (device_name && strcmp(device_name, "")) { - param = pb_DeviceParam_name(strdup(device_name)); - } - else { - char* device_type = getenv("PARBOIL_DEVICE_TYPE"); - if (device_type && strcmp(device_type, "")) { - if (strcmp(device_type, "CPU") == 0) - param = pb_DeviceParam_cpu(); - else if (strcmp(device_type, "GPU") == 0) - param = pb_DeviceParam_gpu(); - else if (strcmp(device_type, "ACCELERATOR") == 0) - param = pb_DeviceParam_accelerator(); - } - } - } - - /* Get a device */ - pb_Device *d = pb_GetDevice(pb_platform, param); - pb_FreeDeviceParam(param); - - return d; -} - -pb_Platform* -pb_GetPlatformByEnvVars() { - char* name = getenv("PARBOIL_PLATFORM_NAME"); - char* version = getenv("PARBOIL_PLATFORM_VERSION"); - - /* Create a pb_PlatformParam object (or NULL) representing the data from the - * environment variables */ - struct pb_PlatformParam *platform; - - if (name) { - if (version) { - platform = pb_PlatformParam(strdup(name), strdup(version)); - } - else { - platform = pb_PlatformParam(strdup(name), NULL); - } - } - else { - platform = NULL; - } - - /* Convert to a platform */ - pb_Platform *p = pb_GetPlatform(platform); - pb_FreePlatformParam(platform); - - return p; -} - -/* Choose an OpenCL platform based on the given command-line parameters. - * If NULL, use the default OpenCL platform. */ -pb_Platform* -pb_GetPlatform(struct pb_PlatformParam *platform) { - if (platform != NULL) { - /* Try to use command-line parameters to choose platform */ - char *name = platform->name; - char *version = platform->version; - - if (!name) { - fprintf(stderr, "Internal error: NULL pointer"); - exit(-1); - } - - if (version) { - pb_Platform* p = pb_GetPlatformByNameAndVersion(name, version); - if (p) return p; - } - - pb_Platform* p = pb_GetPlatformByName(name); - if (p) return p; - } - - pb_Platform* p = pb_GetPlatformByName(NULL); - if (p == NULL) { - fprintf(stderr, "Error: No OpenCL platform in this system. Exiting."); - exit(-1); - } - return p; -} - -//extern void perf_init(); -//extern void mxpa_scheduler_init(); - -pb_Context* -pb_InitOpenCLContext(struct pb_Parameters* parameters) { -#if 0 - pb_Platform* ps = pb_GetPlatform(parameters->platform); - if (!ps) return NULL; - pb_Device* ds = pb_GetDevice(ps, parameters->device); - if (!ds) return NULL; - - /* HERE INITIALIZE TIMER */ - //perf_init(); - //mxpa_scheduler_init(); - - pb_Context* c = createContext(ps, ds); - pb_PrintPlatformInfo(c); - return c; -#endif - cl_int _err; - cl_platform_id platform_id; - cl_device_id device_id; - cl_context context; - clGetPlatformIDs(1, &platform_id, NULL); - clGetDeviceIDs(platform_id, CL_DEVICE_TYPE_DEFAULT, 1, &device_id, NULL); - context = clCreateContext(NULL, 1, &device_id, NULL, NULL, &_err); - - pb_Context* c = (pb_Context*)malloc(sizeof(pb_Context)); - c->clContext = context; - c->clDeviceId = device_id; - c->clPlatformId = platform_id; - c->pb_platform = (pb_Platform*)malloc(sizeof(pb_Platform)); - c->pb_device = (pb_Device*)malloc(sizeof(pb_Device)); - c->pb_platform->devices = (pb_Device**)malloc(sizeof(pb_Device*) * 2); - c->pb_platform->devices[0] = c->pb_device; - c->pb_platform->devices[1] = NULL; - c->pb_platform->contexts = (pb_Context**)malloc(sizeof(pb_Context*) * 2); - c->pb_platform->contexts[0] = c; - c->pb_platform->contexts[1] = NULL; - c->pb_platform->in_use = 1; - c->pb_device->in_use = 1; - return c; -} - -void -pb_ReleaseOpenCLContext(pb_Context* c) { - pb_ReleasePlatforms(); -} - -void -pb_PrintPlatformInfo(pb_Context* c) { - /*pb_Platform** ps = pb_GetPlatforms(); - if (!ps) { - fprintf (stderr, "No platform found"); - return; - } - - printf ("********************************************************\n"); - printf ("DETECTED OPENCL PLATFORMS AND DEVICES:\n"); - printf ("--------------------------------------------------------\n"); - - while (*ps) { - printf ("PLATFORM = %s, %s", (*ps)->name, (*ps)->version); - if (c->pb_platform == *ps) printf (" (SELECTED)"); - printf ("\n"); - - pb_Device** ds = (pb_Device **) pb_GetDevices((*ps)); - if (ds == NULL) { - printf (" + (No devices)\n"); - } else { - while (*ds) { - printf (" + %d: %s", (*ds)->id, (*ds)->name); - if (c->pb_device == *ds) printf (" (SELECTED)"); - printf ("\n"); - ds++; - } - } - - ps++; - } - printf ("********************************************************\n");*/ -} - -#ifdef MEASURE_KERNEL_TIME - -#undef clEnqueueNDRangeKernel - -//extern void pin_trace_enable(char*); -//extern void pin_trace_disable(char*); - -cl_int -pb_clEnqueueNDRangeKernel(cl_command_queue q/* command_queue */, - cl_kernel k/* kernel */, - cl_uint d/* work_dim */, - const size_t * o/* global_work_offset */, - const size_t * gws/* global_work_size */, - const size_t * lws/* local_work_size */, - cl_uint n/* num_events_in_wait_list */, - const cl_event * w/* event_wait_list */, - cl_event * e/* event */) { - - char buf[128]; - struct timeval begin, end; - clGetKernelInfo(k, CL_KERNEL_FUNCTION_NAME, 128, buf, NULL); - -#if 0 - int i; - for (i = 0; i < d; i++) { - printf ("%s: %d: %d / %d\n", buf, i, gws[i], (lws == NULL ? 0 : lws[i])); - } -#endif - - clFinish(q); clFlush(q); - //pin_trace_enable(buf); - //gettimeofday(&begin, NULL); - cl_int result = clEnqueueNDRangeKernel(q, k, d, o, gws, lws, n, w, e); - clFinish(q); clFlush(q); - //gettimeofday(&end, NULL); - //pin_trace_disable(buf); - //float t = (float)(end.tv_sec - begin.tv_sec) + (end.tv_usec - begin.tv_usec) / 1000000.0f; - fflush(stdout); - fflush(stderr); - //printf ("PBTIMER: %s: %f\n", buf, t); - return result; -} - -#endif - -void -pb_sig_float(char* c, float* p, int sz) { - int i; - double s = 0.0; - for (i = 0; i < sz; i++) s += p[i] * (float)(i+1); - printf ("[Signature] %s = %lf\n", c, s); -} - -void -pb_sig_double(char* c, double* p, int sz) { - int i; - double s = 0.0; - for (i = 0; i < sz; i++) s += p[i]; - printf ("[Signature] %s = %lf\n", c, s); -} - -void -pb_sig_short(char* c, short* p, int sz) { - int i; - long long int s = 0; - for (i = 0; i < sz; i++) s += p[i]; - printf ("[Signature] %s = %lld\n", c, s); -} - -void -pb_sig_int(char* c, int* p, int sz) { - int i; - long long int s = 0; - for (i = 0; i < sz; i++) s += p[i]; - printf ("[Signature] %s = %lld\n", c, s); -} - -void -pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { - int i; - unsigned long long int s = 0; - for (i = 0; i < sz; i++) s += p[i]; - printf ("[Signature] %s = %lld\n", c, s); -} - -void pb_sig_clmem(char* s, cl_command_queue command_queue, cl_mem memobj, int ty) { - size_t sz; - if (clGetMemObjectInfo(memobj, CL_MEM_SIZE, sizeof(size_t), &sz, NULL) != CL_SUCCESS) { - printf ("Something wrong.\n"); - assert(0); - } else { - printf ("size = %d\n", sz); - } - char* hp; // = (char*) malloc(sz); - //posix_memalign((void**)&hp, 64, sz); - hp = (char*)malloc(sz); - - clEnqueueReadBuffer (command_queue, - memobj, - CL_TRUE, - 0, - sz, - hp, - 0, - NULL, - NULL); - - if (ty == T_FLOAT) pb_sig_float(s, (float*)hp, sz/sizeof(float)); - if (ty == T_DOUBLE) pb_sig_double(s, (double*)hp, sz/sizeof(double)); - if (ty == T_INT) pb_sig_int(s, (int*)hp, sz/sizeof(int)); - if (ty == T_SHORT) pb_sig_short(s, (short*)hp, sz/sizeof(short)); - if (ty == T_UCHAR) pb_sig_uchar(s, (unsigned char*)hp, sz/sizeof(char)); - - free(hp); -} - diff --git a/tests/opencl/cutcp/readatom.c b/tests/opencl/cutcp/readatom.c deleted file mode 100644 index 8fc89ff3..00000000 --- a/tests/opencl/cutcp/readatom.c +++ /dev/null @@ -1,139 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2008-2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#include -#include -#include -#include -#include "atom.h" - - -#define LINELEN 96 -#define INITLEN 20 - - -Atoms *read_atom_file(const char *fname) -{ - FILE *file; - char line[LINELEN]; - - Atom *atom; /* Atom array */ - int len = INITLEN; /* Size of atom array */ - int cnt = 0; /* Number of atoms read */ - - /* allocate initial atom array */ - atom = (Atom *) malloc(len * sizeof(Atom)); - if (NULL==atom) { - fprintf(stderr, "can't allocate memory\n"); - return NULL; - } - - int i; - for (i = 0; i < len; ++i) { - atom[i].x = i+0; - atom[i].y = i+1; - atom[i].z = i+2; - atom[i].q = 1; - } - -#if 0 - /* open atom "pqr" file */ - file = fopen(fname, "r"); - if (NULL==file) { - fprintf(stderr, "can't open file \"%s\" for reading\n", fname); - return NULL; - } - - /* loop to read pqr file line by line */ - while (fgets(line, LINELEN, file) != NULL) { - - if (strncmp(line, "ATOM ", 6) != 0 && strncmp(line, "HETATM", 6) != 0) { - continue; /* skip anything that isn't an atom record */ - } - - if (cnt==len) { /* extend atom array */ - void *tmp = realloc(atom, 2*len*sizeof(Atom)); - if (NULL==tmp) { - fprintf(stderr, "can't allocate more memory\n"); - return NULL; - } - atom = (Atom *) tmp; - len *= 2; - } - - /* read position coordinates and charge from atom record */ - if (sscanf(line, "%*s %*d %*s %*s %*d %f %f %f %f", &(atom[cnt].x), - &(atom[cnt].y), &(atom[cnt].z), &(atom[cnt].q)) != 4) { - fprintf(stderr, "atom record %d does not have expected format\n", cnt+1); - return NULL; - } - - cnt++; /* count atoms as we store them */ - } - - /* verify EOF and close file */ - if ( !feof(file) ) { - fprintf(stderr, "did not find EOF\n"); - return NULL; - } - if (fclose(file)) { - fprintf(stderr, "can't close file\n"); - return NULL; - } -#endif - - /* Build the output data structure */ - { - Atoms *out = (Atoms *)malloc(sizeof(Atoms)); - - if (NULL == out) { - fprintf(stderr, "can't allocate memory\n"); - return NULL; - } - - out->size = cnt; - out->atoms = atom; - - return out; - } -} - - -void free_atom(Atoms *atom) -{ - if (atom) { - free(atom->atoms); - free(atom); - } -} - -void -get_atom_extent(Vec3 *out_lo, Vec3 *out_hi, Atoms *atom) -{ - Atom *atoms = atom->atoms; - int natoms = atom->size; - Vec3 lo; - Vec3 hi; - int n; - - hi.x = lo.x = atoms[0].x; - hi.y = lo.y = atoms[0].y; - hi.z = lo.z = atoms[0].z; - - for (n = 1; n < natoms; n++) { - lo.x = fminf(lo.x, atoms[n].x); - hi.x = fmaxf(hi.x, atoms[n].x); - lo.y = fminf(lo.y, atoms[n].y); - hi.y = fmaxf(hi.y, atoms[n].y); - lo.z = fminf(lo.z, atoms[n].z); - hi.z = fmaxf(hi.z, atoms[n].z); - } - - *out_lo = lo; - *out_hi = hi; -} diff --git a/tests/opencl/cutcp/watbox.sl40.pqr b/tests/opencl/cutcp/watbox.sl40.pqr deleted file mode 100755 index 917c3faa..00000000 --- a/tests/opencl/cutcp/watbox.sl40.pqr +++ /dev/null @@ -1,5945 +0,0 @@ -CRYST1 0.000 0.000 0.000 90.00 90.00 90.00 P 1 1 -ATOM 1 OH2 TIP3 5 -16.332 -9.918 -4.096 -0.834 1.520 -ATOM 2 H1 TIP3 5 -16.776 -9.549 -4.899 0.417 1.000 -ATOM 3 H2 TIP3 5 -16.908 -9.621 -3.373 0.417 1.000 -ATOM 4 OH2 TIP3 7 -13.967 -15.124 0.891 -0.834 1.520 -ATOM 5 H1 TIP3 7 -13.922 -14.776 1.798 0.417 1.000 -ATOM 6 H2 TIP3 7 -13.408 -15.912 0.961 0.417 1.000 -ATOM 7 OH2 TIP3 8 -15.431 -16.040 9.887 -0.834 1.520 -ATOM 8 H1 TIP3 8 -16.041 -16.840 9.980 0.417 1.000 -ATOM 9 H2 TIP3 8 -16.005 -15.345 9.422 0.417 1.000 -ATOM 10 OH2 TIP3 15 -19.025 -5.993 18.567 -0.834 1.520 -ATOM 11 H1 TIP3 15 -18.863 -6.737 17.966 0.417 1.000 -ATOM 12 H2 TIP3 15 -19.381 -6.461 19.360 0.417 1.000 -ATOM 13 OH2 TIP3 21 -14.814 -3.304 -7.928 -0.834 1.520 -ATOM 14 H1 TIP3 21 -13.917 -3.021 -8.184 0.417 1.000 -ATOM 15 H2 TIP3 21 -14.663 -4.168 -7.510 0.417 1.000 -ATOM 16 OH2 TIP3 26 -5.563 -14.206 10.556 -0.834 1.520 -ATOM 17 H1 TIP3 26 -6.476 -14.681 10.532 0.417 1.000 -ATOM 18 H2 TIP3 26 -5.016 -14.887 10.973 0.417 1.000 -ATOM 19 OH2 TIP3 29 -11.676 -6.189 1.332 -0.834 1.520 -ATOM 20 H1 TIP3 29 -11.367 -6.652 2.112 0.417 1.000 -ATOM 21 H2 TIP3 29 -11.494 -6.884 0.645 0.417 1.000 -ATOM 22 OH2 TIP3 30 -8.538 6.026 11.373 -0.834 1.520 -ATOM 23 H1 TIP3 30 -8.614 6.695 12.076 0.417 1.000 -ATOM 24 H2 TIP3 30 -9.381 6.080 10.936 0.417 1.000 -ATOM 25 OH2 TIP3 44 -16.214 -3.682 -18.690 -0.834 1.520 -ATOM 26 H1 TIP3 44 -16.219 -4.612 -18.804 0.417 1.000 -ATOM 27 H2 TIP3 44 -15.472 -3.420 -19.219 0.417 1.000 -ATOM 28 OH2 TIP3 46 -13.795 -13.009 -1.558 -0.834 1.520 -ATOM 29 H1 TIP3 46 -13.218 -12.653 -0.848 0.417 1.000 -ATOM 30 H2 TIP3 46 -14.691 -12.938 -1.200 0.417 1.000 -ATOM 31 OH2 TIP3 47 -3.874 -5.299 -15.423 -0.834 1.520 -ATOM 32 H1 TIP3 47 -3.930 -5.728 -16.274 0.417 1.000 -ATOM 33 H2 TIP3 47 -3.195 -4.546 -15.555 0.417 1.000 -ATOM 34 OH2 TIP3 48 -17.250 -11.156 -10.371 -0.834 1.520 -ATOM 35 H1 TIP3 48 -17.890 -11.710 -10.002 0.417 1.000 -ATOM 36 H2 TIP3 48 -17.836 -10.755 -11.038 0.417 1.000 -ATOM 37 OH2 TIP3 52 -16.065 -3.805 2.109 -0.834 1.520 -ATOM 38 H1 TIP3 52 -15.130 -3.966 1.985 0.417 1.000 -ATOM 39 H2 TIP3 52 -16.345 -4.633 2.581 0.417 1.000 -ATOM 40 OH2 TIP3 53 -17.009 -8.838 7.860 -0.834 1.520 -ATOM 41 H1 TIP3 53 -16.406 -9.032 7.129 0.417 1.000 -ATOM 42 H2 TIP3 53 -17.586 -8.225 7.430 0.417 1.000 -ATOM 43 OH2 TIP3 55 -15.760 -12.963 7.081 -0.834 1.520 -ATOM 44 H1 TIP3 55 -15.344 -12.169 6.658 0.417 1.000 -ATOM 45 H2 TIP3 55 -16.214 -12.572 7.812 0.417 1.000 -ATOM 46 OH2 TIP3 56 -12.828 -13.001 18.581 -0.834 1.520 -ATOM 47 H1 TIP3 56 -12.960 -12.450 19.327 0.417 1.000 -ATOM 48 H2 TIP3 56 -13.756 -13.244 18.263 0.417 1.000 -ATOM 49 OH2 TIP3 64 -9.326 -11.091 -15.014 -0.834 1.520 -ATOM 50 H1 TIP3 64 -9.002 -11.129 -14.068 0.417 1.000 -ATOM 51 H2 TIP3 64 -10.252 -10.730 -14.925 0.417 1.000 -ATOM 52 OH2 TIP3 67 -19.998 -14.333 -5.409 -0.834 1.520 -ATOM 53 H1 TIP3 67 -19.517 -14.010 -6.170 0.417 1.000 -ATOM 54 H2 TIP3 67 -19.392 -14.219 -4.653 0.417 1.000 -ATOM 55 OH2 TIP3 68 -10.118 -10.844 -5.084 -0.834 1.520 -ATOM 56 H1 TIP3 68 -10.426 -10.247 -4.403 0.417 1.000 -ATOM 57 H2 TIP3 68 -9.979 -11.736 -4.711 0.417 1.000 -ATOM 58 OH2 TIP3 71 -12.036 -13.893 9.453 -0.834 1.520 -ATOM 59 H1 TIP3 71 -11.758 -14.794 9.599 0.417 1.000 -ATOM 60 H2 TIP3 71 -11.578 -13.455 10.237 0.417 1.000 -ATOM 61 OH2 TIP3 75 4.177 -5.729 18.126 -0.834 1.520 -ATOM 62 H1 TIP3 75 3.534 -6.058 17.459 0.417 1.000 -ATOM 63 H2 TIP3 75 5.062 -5.918 17.682 0.417 1.000 -ATOM 64 OH2 TIP3 76 -13.684 -14.712 7.495 -0.834 1.520 -ATOM 65 H1 TIP3 76 -13.360 -14.369 8.363 0.417 1.000 -ATOM 66 H2 TIP3 76 -14.437 -14.148 7.280 0.417 1.000 -ATOM 67 OH2 TIP3 86 -7.111 9.660 -10.913 -0.834 1.520 -ATOM 68 H1 TIP3 86 -7.422 9.796 -10.025 0.417 1.000 -ATOM 69 H2 TIP3 86 -6.236 9.337 -10.788 0.417 1.000 -ATOM 70 OH2 TIP3 87 -18.339 -13.319 -18.057 -0.834 1.520 -ATOM 71 H1 TIP3 87 -18.057 -14.261 -18.019 0.417 1.000 -ATOM 72 H2 TIP3 87 -19.222 -13.401 -18.517 0.417 1.000 -ATOM 73 OH2 TIP3 88 -15.179 0.603 -8.761 -0.834 1.520 -ATOM 74 H1 TIP3 88 -14.534 0.782 -8.037 0.417 1.000 -ATOM 75 H2 TIP3 88 -14.616 0.914 -9.544 0.417 1.000 -ATOM 76 OH2 TIP3 90 -4.978 0.855 -7.169 -0.834 1.520 -ATOM 77 H1 TIP3 90 -5.778 0.809 -6.708 0.417 1.000 -ATOM 78 H2 TIP3 90 -5.327 1.057 -8.082 0.417 1.000 -ATOM 79 OH2 TIP3 94 -18.661 -16.218 2.814 -0.834 1.520 -ATOM 80 H1 TIP3 94 -19.161 -16.575 2.087 0.417 1.000 -ATOM 81 H2 TIP3 94 -18.225 -16.923 3.138 0.417 1.000 -ATOM 82 OH2 TIP3 95 -19.515 -4.863 2.476 -0.834 1.520 -ATOM 83 H1 TIP3 95 -19.356 -4.575 3.349 0.417 1.000 -ATOM 84 H2 TIP3 95 -19.688 -4.039 1.975 0.417 1.000 -ATOM 85 OH2 TIP3 97 -6.857 -6.205 13.479 -0.834 1.520 -ATOM 86 H1 TIP3 97 -7.069 -6.447 14.356 0.417 1.000 -ATOM 87 H2 TIP3 97 -6.477 -5.338 13.519 0.417 1.000 -ATOM 88 OH2 TIP3 98 -7.579 0.595 11.343 -0.834 1.520 -ATOM 89 H1 TIP3 98 -7.670 0.621 12.303 0.417 1.000 -ATOM 90 H2 TIP3 98 -7.719 1.526 11.063 0.417 1.000 -ATOM 91 OH2 TIP3 108 -14.392 3.918 -7.041 -0.834 1.520 -ATOM 92 H1 TIP3 108 -15.160 3.622 -7.455 0.417 1.000 -ATOM 93 H2 TIP3 108 -13.963 4.336 -7.792 0.417 1.000 -ATOM 94 OH2 TIP3 111 -9.574 -12.396 -17.596 -0.834 1.520 -ATOM 95 H1 TIP3 111 -9.239 -11.599 -18.006 0.417 1.000 -ATOM 96 H2 TIP3 111 -9.752 -12.111 -16.692 0.417 1.000 -ATOM 97 OH2 TIP3 112 -11.431 5.684 -7.789 -0.834 1.520 -ATOM 98 H1 TIP3 112 -10.877 5.542 -6.985 0.417 1.000 -ATOM 99 H2 TIP3 112 -11.195 6.595 -8.073 0.417 1.000 -ATOM 100 OH2 TIP3 113 -15.731 2.752 -3.735 -0.834 1.520 -ATOM 101 H1 TIP3 113 -15.851 3.680 -4.112 0.417 1.000 -ATOM 102 H2 TIP3 113 -14.939 2.796 -3.224 0.417 1.000 -ATOM 103 OH2 TIP3 115 -16.401 -6.311 3.583 -0.834 1.520 -ATOM 104 H1 TIP3 115 -16.570 -7.124 4.062 0.417 1.000 -ATOM 105 H2 TIP3 115 -15.439 -6.231 3.661 0.417 1.000 -ATOM 106 OH2 TIP3 116 -0.048 -18.039 6.164 -0.834 1.520 -ATOM 107 H1 TIP3 116 0.414 -18.921 6.079 0.417 1.000 -ATOM 108 H2 TIP3 116 0.667 -17.347 6.013 0.417 1.000 -ATOM 109 OH2 TIP3 125 2.572 -13.758 -16.522 -0.834 1.520 -ATOM 110 H1 TIP3 125 1.705 -14.155 -16.170 0.417 1.000 -ATOM 111 H2 TIP3 125 2.951 -14.341 -17.142 0.417 1.000 -ATOM 112 OH2 TIP3 129 -12.218 8.346 0.641 -0.834 1.520 -ATOM 113 H1 TIP3 129 -11.345 8.234 1.114 0.417 1.000 -ATOM 114 H2 TIP3 129 -12.331 9.295 0.444 0.417 1.000 -ATOM 115 OH2 TIP3 130 -9.074 -14.635 -18.982 -0.834 1.520 -ATOM 116 H1 TIP3 130 -8.862 -15.231 -18.246 0.417 1.000 -ATOM 117 H2 TIP3 130 -9.343 -13.788 -18.507 0.417 1.000 -ATOM 118 OH2 TIP3 131 18.680 4.645 5.756 -0.834 1.520 -ATOM 119 H1 TIP3 131 18.621 5.384 5.135 0.417 1.000 -ATOM 120 H2 TIP3 131 19.060 4.974 6.580 0.417 1.000 -ATOM 121 OH2 TIP3 133 -11.353 -15.686 -0.722 -0.834 1.520 -ATOM 122 H1 TIP3 133 -12.036 -16.389 -0.674 0.417 1.000 -ATOM 123 H2 TIP3 133 -11.957 -14.936 -0.847 0.417 1.000 -ATOM 124 OH2 TIP3 134 -11.619 -2.681 3.275 -0.834 1.520 -ATOM 125 H1 TIP3 134 -12.484 -2.660 3.695 0.417 1.000 -ATOM 126 H2 TIP3 134 -11.569 -1.737 3.009 0.417 1.000 -ATOM 127 OH2 TIP3 135 -15.658 -8.612 5.271 -0.834 1.520 -ATOM 128 H1 TIP3 135 -15.158 -9.303 5.678 0.417 1.000 -ATOM 129 H2 TIP3 135 -15.097 -7.852 5.578 0.417 1.000 -ATOM 130 OH2 TIP3 137 -18.330 -4.325 9.166 -0.834 1.520 -ATOM 131 H1 TIP3 137 -17.880 -4.050 9.974 0.417 1.000 -ATOM 132 H2 TIP3 137 -19.237 -4.483 9.468 0.417 1.000 -ATOM 133 OH2 TIP3 145 -3.530 8.089 -17.819 -0.834 1.520 -ATOM 134 H1 TIP3 145 -3.146 7.230 -17.501 0.417 1.000 -ATOM 135 H2 TIP3 145 -4.467 7.856 -17.912 0.417 1.000 -ATOM 136 OH2 TIP3 146 -11.624 12.914 -11.264 -0.834 1.520 -ATOM 137 H1 TIP3 146 -10.900 12.283 -11.266 0.417 1.000 -ATOM 138 H2 TIP3 146 -11.193 13.795 -11.432 0.417 1.000 -ATOM 139 OH2 TIP3 149 -19.198 5.084 -14.477 -0.834 1.520 -ATOM 140 H1 TIP3 149 -19.421 6.065 -14.495 0.417 1.000 -ATOM 141 H2 TIP3 149 -18.205 5.163 -14.555 0.417 1.000 -ATOM 142 OH2 TIP3 150 -15.701 -2.226 -12.831 -0.834 1.520 -ATOM 143 H1 TIP3 150 -16.697 -2.197 -12.678 0.417 1.000 -ATOM 144 H2 TIP3 150 -15.467 -1.245 -13.076 0.417 1.000 -ATOM 145 OH2 TIP3 152 -11.108 -7.023 -14.679 -0.834 1.520 -ATOM 146 H1 TIP3 152 -11.881 -7.522 -14.557 0.417 1.000 -ATOM 147 H2 TIP3 152 -10.658 -7.443 -15.464 0.417 1.000 -ATOM 148 OH2 TIP3 153 -17.292 0.969 4.504 -0.834 1.520 -ATOM 149 H1 TIP3 153 -17.411 0.945 3.556 0.417 1.000 -ATOM 150 H2 TIP3 153 -16.877 1.835 4.549 0.417 1.000 -ATOM 151 OH2 TIP3 155 -17.572 -7.269 10.487 -0.834 1.520 -ATOM 152 H1 TIP3 155 -17.281 -7.927 9.830 0.417 1.000 -ATOM 153 H2 TIP3 155 -17.418 -6.458 10.008 0.417 1.000 -ATOM 154 OH2 TIP3 156 -8.719 11.586 4.226 -0.834 1.520 -ATOM 155 H1 TIP3 156 -8.853 10.638 4.048 0.417 1.000 -ATOM 156 H2 TIP3 156 -7.869 11.561 4.702 0.417 1.000 -ATOM 157 OH2 TIP3 167 -17.826 -1.532 -17.788 -0.834 1.520 -ATOM 158 H1 TIP3 167 -17.368 -2.372 -17.797 0.417 1.000 -ATOM 159 H2 TIP3 167 -17.069 -0.880 -17.719 0.417 1.000 -ATOM 160 OH2 TIP3 177 -14.952 -10.954 16.578 -0.834 1.520 -ATOM 161 H1 TIP3 177 -14.936 -10.111 17.059 0.417 1.000 -ATOM 162 H2 TIP3 177 -15.378 -10.776 15.743 0.417 1.000 -ATOM 163 OH2 TIP3 178 -15.182 16.858 -2.213 -0.834 1.520 -ATOM 164 H1 TIP3 178 -15.854 16.220 -2.533 0.417 1.000 -ATOM 165 H2 TIP3 178 -15.016 16.600 -1.315 0.417 1.000 -ATOM 166 OH2 TIP3 179 -10.414 15.305 16.263 -0.834 1.520 -ATOM 167 H1 TIP3 179 -10.047 15.826 17.009 0.417 1.000 -ATOM 168 H2 TIP3 179 -9.737 15.511 15.531 0.417 1.000 -ATOM 169 OH2 TIP3 180 -7.610 7.307 17.360 -0.834 1.520 -ATOM 170 H1 TIP3 180 -8.155 8.113 17.293 0.417 1.000 -ATOM 171 H2 TIP3 180 -7.955 6.920 18.175 0.417 1.000 -ATOM 172 OH2 TIP3 191 -18.669 16.142 -17.299 -0.834 1.520 -ATOM 173 H1 TIP3 191 -19.635 16.131 -17.037 0.417 1.000 -ATOM 174 H2 TIP3 191 -18.328 15.284 -16.967 0.417 1.000 -ATOM 175 OH2 TIP3 195 -17.084 11.913 -11.022 -0.834 1.520 -ATOM 176 H1 TIP3 195 -16.227 12.387 -10.984 0.417 1.000 -ATOM 177 H2 TIP3 195 -17.775 12.579 -10.972 0.417 1.000 -ATOM 178 OH2 TIP3 197 -1.130 1.460 8.971 -0.834 1.520 -ATOM 179 H1 TIP3 197 -2.097 1.288 8.721 0.417 1.000 -ATOM 180 H2 TIP3 197 -0.926 0.691 9.591 0.417 1.000 -ATOM 181 OH2 TIP3 198 -13.921 13.356 5.597 -0.834 1.520 -ATOM 182 H1 TIP3 198 -13.517 13.937 6.265 0.417 1.000 -ATOM 183 H2 TIP3 198 -14.818 13.791 5.496 0.417 1.000 -ATOM 184 OH2 TIP3 201 -16.478 9.694 7.275 -0.834 1.520 -ATOM 185 H1 TIP3 201 -17.021 9.239 8.035 0.417 1.000 -ATOM 186 H2 TIP3 201 -17.110 10.250 6.887 0.417 1.000 -ATOM 187 OH2 TIP3 202 -4.656 14.738 12.731 -0.834 1.520 -ATOM 188 H1 TIP3 202 -4.474 15.545 13.160 0.417 1.000 -ATOM 189 H2 TIP3 202 -5.087 15.025 11.884 0.417 1.000 -ATOM 190 OH2 TIP3 212 -10.936 10.446 -2.304 -0.834 1.520 -ATOM 191 H1 TIP3 212 -10.104 10.192 -1.919 0.417 1.000 -ATOM 192 H2 TIP3 212 -11.502 10.631 -1.509 0.417 1.000 -ATOM 193 OH2 TIP3 218 -1.580 12.007 2.329 -0.834 1.520 -ATOM 194 H1 TIP3 218 -2.087 11.810 3.089 0.417 1.000 -ATOM 195 H2 TIP3 218 -0.655 12.190 2.662 0.417 1.000 -ATOM 196 OH2 TIP3 222 -5.630 16.170 16.010 -0.834 1.520 -ATOM 197 H1 TIP3 222 -5.210 16.753 15.304 0.417 1.000 -ATOM 198 H2 TIP3 222 -4.931 15.483 16.044 0.417 1.000 -ATOM 199 OH2 TIP3 237 -19.435 9.252 -18.417 -0.834 1.520 -ATOM 200 H1 TIP3 237 -18.537 9.562 -18.264 0.417 1.000 -ATOM 201 H2 TIP3 237 -19.414 8.335 -18.212 0.417 1.000 -ATOM 202 OH2 TIP3 238 -6.261 13.516 17.538 -0.834 1.520 -ATOM 203 H1 TIP3 238 -6.658 13.364 16.663 0.417 1.000 -ATOM 204 H2 TIP3 238 -6.871 12.963 18.063 0.417 1.000 -ATOM 205 OH2 TIP3 239 -6.388 -2.320 -1.378 -0.834 1.520 -ATOM 206 H1 TIP3 239 -6.256 -2.586 -0.492 0.417 1.000 -ATOM 207 H2 TIP3 239 -5.474 -2.169 -1.653 0.417 1.000 -ATOM 208 OH2 TIP3 252 -10.142 18.394 -14.444 -0.834 1.520 -ATOM 209 H1 TIP3 252 -10.914 18.132 -14.008 0.417 1.000 -ATOM 210 H2 TIP3 252 -9.715 19.003 -13.917 0.417 1.000 -ATOM 211 OH2 TIP3 253 -9.918 14.674 -8.066 -0.834 1.520 -ATOM 212 H1 TIP3 253 -9.792 15.123 -7.205 0.417 1.000 -ATOM 213 H2 TIP3 253 -10.552 15.288 -8.455 0.417 1.000 -ATOM 214 OH2 TIP3 257 -14.115 15.212 -12.437 -0.834 1.520 -ATOM 215 H1 TIP3 257 -14.491 15.849 -13.109 0.417 1.000 -ATOM 216 H2 TIP3 257 -13.198 15.079 -12.688 0.417 1.000 -ATOM 217 OH2 TIP3 261 -18.713 4.132 3.737 -0.834 1.520 -ATOM 218 H1 TIP3 261 -18.765 3.924 4.731 0.417 1.000 -ATOM 219 H2 TIP3 261 -17.912 4.709 3.639 0.417 1.000 -ATOM 220 OH2 TIP3 281 -13.418 17.780 7.400 -0.834 1.520 -ATOM 221 H1 TIP3 281 -13.160 16.904 7.215 0.417 1.000 -ATOM 222 H2 TIP3 281 -12.817 18.042 8.140 0.417 1.000 -ATOM 223 OH2 TIP3 295 -11.916 12.876 -4.176 -0.834 1.520 -ATOM 224 H1 TIP3 295 -12.857 12.955 -4.219 0.417 1.000 -ATOM 225 H2 TIP3 295 -11.723 12.234 -3.503 0.417 1.000 -ATOM 226 OH2 TIP3 365 -13.635 -17.326 8.059 -0.834 1.520 -ATOM 227 H1 TIP3 365 -14.201 -16.563 7.849 0.417 1.000 -ATOM 228 H2 TIP3 365 -12.825 -16.945 8.403 0.417 1.000 -ATOM 229 OH2 TIP3 411 -10.661 -13.262 11.901 -0.834 1.520 -ATOM 230 H1 TIP3 411 -11.096 -12.764 12.634 0.417 1.000 -ATOM 231 H2 TIP3 411 -10.596 -14.162 12.222 0.417 1.000 -ATOM 232 OH2 TIP3 424 -4.586 -13.431 -17.499 -0.834 1.520 -ATOM 233 H1 TIP3 424 -3.776 -13.142 -17.872 0.417 1.000 -ATOM 234 H2 TIP3 424 -5.192 -13.300 -18.329 0.417 1.000 -ATOM 235 OH2 TIP3 432 -12.429 -16.202 14.095 -0.834 1.520 -ATOM 236 H1 TIP3 432 -13.034 -15.596 13.494 0.417 1.000 -ATOM 237 H2 TIP3 432 -12.554 -15.905 15.028 0.417 1.000 -ATOM 238 OH2 TIP3 443 -10.065 -7.124 -9.809 -0.834 1.520 -ATOM 239 H1 TIP3 443 -10.291 -7.902 -10.366 0.417 1.000 -ATOM 240 H2 TIP3 443 -10.528 -7.234 -8.982 0.417 1.000 -ATOM 241 OH2 TIP3 450 -12.940 -17.926 -0.170 -0.834 1.520 -ATOM 242 H1 TIP3 450 -12.350 -18.381 0.528 0.417 1.000 -ATOM 243 H2 TIP3 450 -13.852 -18.204 0.035 0.417 1.000 -ATOM 244 OH2 TIP3 452 -9.105 -0.675 19.064 -0.834 1.520 -ATOM 245 H1 TIP3 452 -9.633 -1.232 19.669 0.417 1.000 -ATOM 246 H2 TIP3 452 -8.214 -1.042 19.229 0.417 1.000 -ATOM 247 OH2 TIP3 465 -6.109 -18.081 -17.008 -0.834 1.520 -ATOM 248 H1 TIP3 465 -5.802 -17.319 -16.578 0.417 1.000 -ATOM 249 H2 TIP3 465 -6.788 -18.423 -16.422 0.417 1.000 -ATOM 250 OH2 TIP3 466 -11.177 -9.419 -11.173 -0.834 1.520 -ATOM 251 H1 TIP3 466 -11.817 -9.247 -11.852 0.417 1.000 -ATOM 252 H2 TIP3 466 -11.611 -9.939 -10.466 0.417 1.000 -ATOM 253 OH2 TIP3 469 -10.922 -8.033 -7.266 -0.834 1.520 -ATOM 254 H1 TIP3 469 -11.772 -8.439 -7.007 0.417 1.000 -ATOM 255 H2 TIP3 469 -10.488 -7.806 -6.440 0.417 1.000 -ATOM 256 OH2 TIP3 484 -9.718 -19.360 -17.947 -0.834 1.520 -ATOM 257 H1 TIP3 484 -9.633 -19.041 -16.984 0.417 1.000 -ATOM 258 H2 TIP3 484 -10.257 -18.677 -18.333 0.417 1.000 -ATOM 259 OH2 TIP3 485 -15.382 -11.875 -12.223 -0.834 1.520 -ATOM 260 H1 TIP3 485 -14.595 -12.081 -11.686 0.417 1.000 -ATOM 261 H2 TIP3 485 -15.998 -11.542 -11.578 0.417 1.000 -ATOM 262 OH2 TIP3 486 -16.597 -15.313 -10.185 -0.834 1.520 -ATOM 263 H1 TIP3 486 -16.891 -14.728 -10.903 0.417 1.000 -ATOM 264 H2 TIP3 486 -16.183 -16.027 -10.673 0.417 1.000 -ATOM 265 OH2 TIP3 489 -16.724 -12.535 -0.675 -0.834 1.520 -ATOM 266 H1 TIP3 489 -17.228 -13.009 -0.036 0.417 1.000 -ATOM 267 H2 TIP3 489 -16.396 -11.712 -0.247 0.417 1.000 -ATOM 268 OH2 TIP3 490 -17.109 -14.956 12.613 -0.834 1.520 -ATOM 269 H1 TIP3 490 -17.740 -15.480 13.150 0.417 1.000 -ATOM 270 H2 TIP3 490 -17.697 -14.490 12.020 0.417 1.000 -ATOM 271 OH2 TIP3 491 -18.160 -11.735 8.662 -0.834 1.520 -ATOM 272 H1 TIP3 491 -18.320 -11.956 7.747 0.417 1.000 -ATOM 273 H2 TIP3 491 -18.166 -10.789 8.733 0.417 1.000 -ATOM 274 OH2 TIP3 494 -19.263 -2.098 17.090 -0.834 1.520 -ATOM 275 H1 TIP3 494 -19.541 -1.208 17.352 0.417 1.000 -ATOM 276 H2 TIP3 494 -19.134 -2.051 16.147 0.417 1.000 -ATOM 277 OH2 TIP3 495 -10.022 -12.125 4.948 -0.834 1.520 -ATOM 278 H1 TIP3 495 -10.411 -12.374 5.845 0.417 1.000 -ATOM 279 H2 TIP3 495 -9.564 -11.344 5.187 0.417 1.000 -ATOM 280 OH2 TIP3 498 -10.073 -15.815 12.805 -0.834 1.520 -ATOM 281 H1 TIP3 498 -10.944 -16.065 13.202 0.417 1.000 -ATOM 282 H2 TIP3 498 -9.640 -16.675 12.665 0.417 1.000 -ATOM 283 OH2 TIP3 503 -16.881 -9.675 -6.842 -0.834 1.520 -ATOM 284 H1 TIP3 503 -16.402 -9.100 -7.488 0.417 1.000 -ATOM 285 H2 TIP3 503 -16.560 -10.570 -7.038 0.417 1.000 -ATOM 286 OH2 TIP3 506 -17.622 -14.194 -3.747 -0.834 1.520 -ATOM 287 H1 TIP3 506 -17.843 -13.839 -2.901 0.417 1.000 -ATOM 288 H2 TIP3 506 -16.699 -13.968 -3.874 0.417 1.000 -ATOM 289 OH2 TIP3 507 -13.618 2.271 -17.681 -0.834 1.520 -ATOM 290 H1 TIP3 507 -14.450 2.035 -18.153 0.417 1.000 -ATOM 291 H2 TIP3 507 -13.141 1.472 -17.808 0.417 1.000 -ATOM 292 OH2 TIP3 508 11.804 -7.244 -12.948 -0.834 1.520 -ATOM 293 H1 TIP3 508 11.543 -6.358 -13.269 0.417 1.000 -ATOM 294 H2 TIP3 508 12.291 -7.070 -12.133 0.417 1.000 -ATOM 295 OH2 TIP3 509 -15.983 -16.095 -0.582 -0.834 1.520 -ATOM 296 H1 TIP3 509 -16.858 -15.958 -0.868 0.417 1.000 -ATOM 297 H2 TIP3 509 -15.418 -15.273 -0.705 0.417 1.000 -ATOM 298 OH2 TIP3 511 -17.656 0.472 1.631 -0.834 1.520 -ATOM 299 H1 TIP3 511 -17.068 0.821 0.945 0.417 1.000 -ATOM 300 H2 TIP3 511 -18.533 0.548 1.138 0.417 1.000 -ATOM 301 OH2 TIP3 512 0.613 -6.384 -8.934 -0.834 1.520 -ATOM 302 H1 TIP3 512 0.739 -6.041 -9.782 0.417 1.000 -ATOM 303 H2 TIP3 512 -0.381 -6.184 -8.750 0.417 1.000 -ATOM 304 OH2 TIP3 514 -8.508 -14.452 14.846 -0.834 1.520 -ATOM 305 H1 TIP3 514 -8.837 -14.796 14.040 0.417 1.000 -ATOM 306 H2 TIP3 514 -8.614 -13.521 14.859 0.417 1.000 -ATOM 307 OH2 TIP3 515 -18.213 -10.482 4.515 -0.834 1.520 -ATOM 308 H1 TIP3 515 -17.868 -10.425 3.646 0.417 1.000 -ATOM 309 H2 TIP3 515 -18.140 -9.594 4.787 0.417 1.000 -ATOM 310 OH2 TIP3 516 -15.873 -18.130 12.761 -0.834 1.520 -ATOM 311 H1 TIP3 516 -16.170 -17.728 11.989 0.417 1.000 -ATOM 312 H2 TIP3 516 -15.270 -17.438 13.098 0.417 1.000 -ATOM 313 OH2 TIP3 517 -5.474 -9.468 0.137 -0.834 1.520 -ATOM 314 H1 TIP3 517 -5.838 -10.267 0.503 0.417 1.000 -ATOM 315 H2 TIP3 517 -5.534 -9.580 -0.806 0.417 1.000 -ATOM 316 OH2 TIP3 528 6.124 -3.406 -3.193 -0.834 1.520 -ATOM 317 H1 TIP3 528 6.948 -3.136 -2.756 0.417 1.000 -ATOM 318 H2 TIP3 528 6.137 -2.952 -4.100 0.417 1.000 -ATOM 319 OH2 TIP3 529 -11.124 8.437 -14.421 -0.834 1.520 -ATOM 320 H1 TIP3 529 -10.773 7.577 -14.642 0.417 1.000 -ATOM 321 H2 TIP3 529 -11.210 8.398 -13.467 0.417 1.000 -ATOM 322 OH2 TIP3 533 -14.427 -5.196 -0.910 -0.834 1.520 -ATOM 323 H1 TIP3 533 -14.041 -4.661 -0.192 0.417 1.000 -ATOM 324 H2 TIP3 533 -14.131 -4.587 -1.669 0.417 1.000 -ATOM 325 OH2 TIP3 534 -3.717 -12.646 2.781 -0.834 1.520 -ATOM 326 H1 TIP3 534 -4.360 -11.942 2.449 0.417 1.000 -ATOM 327 H2 TIP3 534 -4.027 -13.440 2.379 0.417 1.000 -ATOM 328 OH2 TIP3 535 2.872 -1.559 -3.212 -0.834 1.520 -ATOM 329 H1 TIP3 535 2.201 -2.136 -3.621 0.417 1.000 -ATOM 330 H2 TIP3 535 2.335 -1.019 -2.575 0.417 1.000 -ATOM 331 OH2 TIP3 538 -9.895 -19.159 19.111 -0.834 1.520 -ATOM 332 H1 TIP3 538 -10.512 -18.550 18.631 0.417 1.000 -ATOM 333 H2 TIP3 538 -9.368 -18.582 19.705 0.417 1.000 -ATOM 334 OH2 TIP3 540 -19.258 0.647 17.424 -0.834 1.520 -ATOM 335 H1 TIP3 540 -19.837 1.202 16.855 0.417 1.000 -ATOM 336 H2 TIP3 540 -18.468 0.410 16.837 0.417 1.000 -ATOM 337 OH2 TIP3 543 -14.542 -6.903 -15.411 -0.834 1.520 -ATOM 338 H1 TIP3 543 -14.327 -6.981 -16.388 0.417 1.000 -ATOM 339 H2 TIP3 543 -15.101 -6.106 -15.343 0.417 1.000 -ATOM 340 OH2 TIP3 549 7.849 -10.132 -0.827 -0.834 1.520 -ATOM 341 H1 TIP3 549 7.170 -10.280 -0.137 0.417 1.000 -ATOM 342 H2 TIP3 549 7.403 -9.581 -1.511 0.417 1.000 -ATOM 343 OH2 TIP3 550 -15.448 -18.630 0.330 -0.834 1.520 -ATOM 344 H1 TIP3 550 -15.726 -17.806 -0.047 0.417 1.000 -ATOM 345 H2 TIP3 550 -15.765 -19.199 -0.429 0.417 1.000 -ATOM 346 OH2 TIP3 551 -15.406 5.266 -4.800 -0.834 1.520 -ATOM 347 H1 TIP3 551 -16.362 5.500 -4.804 0.417 1.000 -ATOM 348 H2 TIP3 551 -15.242 4.897 -5.688 0.417 1.000 -ATOM 349 OH2 TIP3 552 -9.034 -19.513 7.664 -0.834 1.520 -ATOM 350 H1 TIP3 552 -9.165 -18.577 7.475 0.417 1.000 -ATOM 351 H2 TIP3 552 -9.442 -19.908 6.883 0.417 1.000 -ATOM 352 OH2 TIP3 553 -13.420 -7.008 9.691 -0.834 1.520 -ATOM 353 H1 TIP3 553 -13.570 -6.114 9.972 0.417 1.000 -ATOM 354 H2 TIP3 553 -13.571 -7.501 10.481 0.417 1.000 -ATOM 355 OH2 TIP3 554 -14.840 -6.256 -12.046 -0.834 1.520 -ATOM 356 H1 TIP3 554 -13.952 -6.578 -11.793 0.417 1.000 -ATOM 357 H2 TIP3 554 -14.992 -5.507 -11.472 0.417 1.000 -ATOM 358 OH2 TIP3 555 -18.512 -12.246 11.838 -0.834 1.520 -ATOM 359 H1 TIP3 555 -19.222 -11.677 12.195 0.417 1.000 -ATOM 360 H2 TIP3 555 -18.457 -11.957 10.921 0.417 1.000 -ATOM 361 OH2 TIP3 558 -9.610 -13.614 -1.172 -0.834 1.520 -ATOM 362 H1 TIP3 558 -9.312 -13.516 -0.269 0.417 1.000 -ATOM 363 H2 TIP3 558 -10.156 -14.411 -1.079 0.417 1.000 -ATOM 364 OH2 TIP3 562 -18.322 1.942 -18.965 -0.834 1.520 -ATOM 365 H1 TIP3 562 -18.728 1.280 -18.408 0.417 1.000 -ATOM 366 H2 TIP3 562 -18.502 2.809 -18.507 0.417 1.000 -ATOM 367 OH2 TIP3 568 -10.370 1.547 -17.000 -0.834 1.520 -ATOM 368 H1 TIP3 568 -10.399 2.471 -17.357 0.417 1.000 -ATOM 369 H2 TIP3 568 -10.713 0.954 -17.688 0.417 1.000 -ATOM 370 OH2 TIP3 570 -19.245 14.044 -7.077 -0.834 1.520 -ATOM 371 H1 TIP3 570 -18.622 14.200 -6.344 0.417 1.000 -ATOM 372 H2 TIP3 570 -19.669 13.221 -6.771 0.417 1.000 -ATOM 373 OH2 TIP3 573 -15.609 9.423 1.120 -0.834 1.520 -ATOM 374 H1 TIP3 573 -16.237 8.799 1.511 0.417 1.000 -ATOM 375 H2 TIP3 573 -15.406 9.044 0.208 0.417 1.000 -ATOM 376 OH2 TIP3 575 -17.656 -9.429 -17.376 -0.834 1.520 -ATOM 377 H1 TIP3 575 -17.878 -9.247 -16.447 0.417 1.000 -ATOM 378 H2 TIP3 575 -17.053 -10.209 -17.383 0.417 1.000 -ATOM 379 OH2 TIP3 576 -17.418 7.635 2.063 -0.834 1.520 -ATOM 380 H1 TIP3 576 -18.174 7.334 2.610 0.417 1.000 -ATOM 381 H2 TIP3 576 -17.859 7.873 1.227 0.417 1.000 -ATOM 382 OH2 TIP3 578 -17.381 8.047 17.820 -0.834 1.520 -ATOM 383 H1 TIP3 578 -17.531 8.994 17.611 0.417 1.000 -ATOM 384 H2 TIP3 578 -18.029 7.756 18.500 0.417 1.000 -ATOM 385 OH2 TIP3 579 -11.877 -12.110 14.105 -0.834 1.520 -ATOM 386 H1 TIP3 579 -11.717 -12.461 15.039 0.417 1.000 -ATOM 387 H2 TIP3 579 -11.271 -11.366 14.000 0.417 1.000 -ATOM 388 OH2 TIP3 580 -6.615 -10.231 9.453 -0.834 1.520 -ATOM 389 H1 TIP3 580 -6.144 -9.609 10.062 0.417 1.000 -ATOM 390 H2 TIP3 580 -7.401 -10.487 9.965 0.417 1.000 -ATOM 391 OH2 TIP3 584 -19.177 3.668 -17.112 -0.834 1.520 -ATOM 392 H1 TIP3 584 -18.364 3.316 -16.660 0.417 1.000 -ATOM 393 H2 TIP3 584 -19.747 3.978 -16.360 0.417 1.000 -ATOM 394 OH2 TIP3 591 -17.917 8.332 -12.674 -0.834 1.520 -ATOM 395 H1 TIP3 591 -18.442 8.514 -11.885 0.417 1.000 -ATOM 396 H2 TIP3 591 -18.504 7.737 -13.189 0.417 1.000 -ATOM 397 OH2 TIP3 592 -6.780 4.494 -6.908 -0.834 1.520 -ATOM 398 H1 TIP3 592 -7.109 5.383 -6.713 0.417 1.000 -ATOM 399 H2 TIP3 592 -6.944 3.918 -6.147 0.417 1.000 -ATOM 400 OH2 TIP3 593 -19.005 8.021 -9.985 -0.834 1.520 -ATOM 401 H1 TIP3 593 -19.987 7.981 -9.957 0.417 1.000 -ATOM 402 H2 TIP3 593 -18.702 7.119 -9.737 0.417 1.000 -ATOM 403 OH2 TIP3 594 -17.290 -13.815 -12.460 -0.834 1.520 -ATOM 404 H1 TIP3 594 -17.533 -13.764 -13.439 0.417 1.000 -ATOM 405 H2 TIP3 594 -16.590 -13.145 -12.420 0.417 1.000 -ATOM 406 OH2 TIP3 595 -13.089 -0.542 -2.665 -0.834 1.520 -ATOM 407 H1 TIP3 595 -12.483 0.118 -3.043 0.417 1.000 -ATOM 408 H2 TIP3 595 -13.685 -0.864 -3.412 0.417 1.000 -ATOM 409 OH2 TIP3 598 -6.664 4.461 16.785 -0.834 1.520 -ATOM 410 H1 TIP3 598 -6.761 5.164 17.434 0.417 1.000 -ATOM 411 H2 TIP3 598 -6.307 3.713 17.244 0.417 1.000 -ATOM 412 OH2 TIP3 599 -17.166 -4.957 14.374 -0.834 1.520 -ATOM 413 H1 TIP3 599 -17.721 -5.193 15.123 0.417 1.000 -ATOM 414 H2 TIP3 599 -16.393 -5.451 14.503 0.417 1.000 -ATOM 415 OH2 TIP3 602 -14.821 0.890 7.387 -0.834 1.520 -ATOM 416 H1 TIP3 602 -15.382 1.606 7.646 0.417 1.000 -ATOM 417 H2 TIP3 602 -14.804 0.211 8.107 0.417 1.000 -ATOM 418 OH2 TIP3 611 -10.661 18.413 -1.977 -0.834 1.520 -ATOM 419 H1 TIP3 611 -10.709 17.725 -1.291 0.417 1.000 -ATOM 420 H2 TIP3 611 -9.895 18.114 -2.489 0.417 1.000 -ATOM 421 OH2 TIP3 612 -3.808 2.403 -16.218 -0.834 1.520 -ATOM 422 H1 TIP3 612 -3.637 2.881 -15.378 0.417 1.000 -ATOM 423 H2 TIP3 612 -3.573 3.065 -16.914 0.417 1.000 -ATOM 424 OH2 TIP3 613 -6.460 2.220 -17.252 -0.834 1.520 -ATOM 425 H1 TIP3 613 -7.035 2.475 -16.538 0.417 1.000 -ATOM 426 H2 TIP3 613 -5.572 2.404 -16.926 0.417 1.000 -ATOM 427 OH2 TIP3 615 -10.475 8.541 10.151 -0.834 1.520 -ATOM 428 H1 TIP3 615 -9.585 8.418 9.686 0.417 1.000 -ATOM 429 H2 TIP3 615 -10.803 7.671 10.239 0.417 1.000 -ATOM 430 OH2 TIP3 616 -16.995 13.969 -5.251 -0.834 1.520 -ATOM 431 H1 TIP3 616 -16.251 14.208 -5.841 0.417 1.000 -ATOM 432 H2 TIP3 616 -16.897 13.042 -5.075 0.417 1.000 -ATOM 433 OH2 TIP3 618 -11.611 -1.835 10.992 -0.834 1.520 -ATOM 434 H1 TIP3 618 -12.032 -2.230 10.210 0.417 1.000 -ATOM 435 H2 TIP3 618 -11.583 -2.549 11.641 0.417 1.000 -ATOM 436 OH2 TIP3 619 -14.803 17.740 0.512 -0.834 1.520 -ATOM 437 H1 TIP3 619 -14.483 18.488 0.028 0.417 1.000 -ATOM 438 H2 TIP3 619 -15.762 17.952 0.773 0.417 1.000 -ATOM 439 OH2 TIP3 624 -4.674 6.968 11.402 -0.834 1.520 -ATOM 440 H1 TIP3 624 -5.547 6.736 10.928 0.417 1.000 -ATOM 441 H2 TIP3 624 -4.932 7.620 12.033 0.417 1.000 -ATOM 442 OH2 TIP3 632 -11.819 3.465 -12.363 -0.834 1.520 -ATOM 443 H1 TIP3 632 -10.993 3.937 -12.297 0.417 1.000 -ATOM 444 H2 TIP3 632 -12.483 4.117 -12.688 0.417 1.000 -ATOM 445 OH2 TIP3 634 -9.071 -5.250 -14.282 -0.834 1.520 -ATOM 446 H1 TIP3 634 -9.950 -5.459 -14.592 0.417 1.000 -ATOM 447 H2 TIP3 634 -8.599 -6.121 -14.283 0.417 1.000 -ATOM 448 OH2 TIP3 635 -0.341 5.552 -2.551 -0.834 1.520 -ATOM 449 H1 TIP3 635 0.182 6.346 -2.501 0.417 1.000 -ATOM 450 H2 TIP3 635 -1.266 5.858 -2.619 0.417 1.000 -ATOM 451 OH2 TIP3 636 -13.016 14.157 3.048 -0.834 1.520 -ATOM 452 H1 TIP3 636 -13.782 14.085 2.457 0.417 1.000 -ATOM 453 H2 TIP3 636 -13.284 13.608 3.800 0.417 1.000 -ATOM 454 OH2 TIP3 637 -13.003 14.755 15.342 -0.834 1.520 -ATOM 455 H1 TIP3 637 -12.400 14.991 15.995 0.417 1.000 -ATOM 456 H2 TIP3 637 -13.065 13.804 15.464 0.417 1.000 -ATOM 457 OH2 TIP3 638 -17.914 5.452 -1.806 -0.834 1.520 -ATOM 458 H1 TIP3 638 -17.826 5.923 -2.606 0.417 1.000 -ATOM 459 H2 TIP3 638 -18.728 5.869 -1.396 0.417 1.000 -ATOM 460 OH2 TIP3 639 -18.317 5.667 6.332 -0.834 1.520 -ATOM 461 H1 TIP3 639 -17.420 5.907 6.426 0.417 1.000 -ATOM 462 H2 TIP3 639 -18.515 5.227 7.126 0.417 1.000 -ATOM 463 OH2 TIP3 640 3.331 1.619 -1.931 -0.834 1.520 -ATOM 464 H1 TIP3 640 3.137 2.516 -1.544 0.417 1.000 -ATOM 465 H2 TIP3 640 2.663 0.985 -1.550 0.417 1.000 -ATOM 466 OH2 TIP3 642 -18.246 14.617 14.408 -0.834 1.520 -ATOM 467 H1 TIP3 642 -18.209 14.662 15.420 0.417 1.000 -ATOM 468 H2 TIP3 642 -17.432 15.011 14.179 0.417 1.000 -ATOM 469 OH2 TIP3 643 -5.300 9.177 13.075 -0.834 1.520 -ATOM 470 H1 TIP3 643 -5.425 8.887 13.996 0.417 1.000 -ATOM 471 H2 TIP3 643 -6.231 9.183 12.748 0.417 1.000 -ATOM 472 OH2 TIP3 657 -10.914 17.628 -9.139 -0.834 1.520 -ATOM 473 H1 TIP3 657 -11.399 18.463 -9.333 0.417 1.000 -ATOM 474 H2 TIP3 657 -10.805 17.683 -8.136 0.417 1.000 -ATOM 475 OH2 TIP3 659 -3.493 -2.830 1.006 -0.834 1.520 -ATOM 476 H1 TIP3 659 -2.774 -2.994 0.295 0.417 1.000 -ATOM 477 H2 TIP3 659 -3.970 -3.703 1.082 0.417 1.000 -ATOM 478 OH2 TIP3 661 -8.069 16.535 12.386 -0.834 1.520 -ATOM 479 H1 TIP3 661 -8.561 16.173 13.148 0.417 1.000 -ATOM 480 H2 TIP3 661 -7.757 15.723 11.845 0.417 1.000 -ATOM 481 OH2 TIP3 662 -16.210 3.318 6.067 -0.834 1.520 -ATOM 482 H1 TIP3 662 -15.889 3.684 6.901 0.417 1.000 -ATOM 483 H2 TIP3 662 -17.165 3.196 6.248 0.417 1.000 -ATOM 484 OH2 TIP3 666 -18.375 10.100 16.358 -0.834 1.520 -ATOM 485 H1 TIP3 666 -18.272 10.712 15.618 0.417 1.000 -ATOM 486 H2 TIP3 666 -19.088 10.549 16.843 0.417 1.000 -ATOM 487 OH2 TIP3 674 -14.786 15.728 -18.104 -0.834 1.520 -ATOM 488 H1 TIP3 674 -14.253 15.733 -18.902 0.417 1.000 -ATOM 489 H2 TIP3 674 -15.411 16.462 -18.332 0.417 1.000 -ATOM 490 OH2 TIP3 676 1.955 12.145 -7.880 -0.834 1.520 -ATOM 491 H1 TIP3 676 2.028 12.757 -7.174 0.417 1.000 -ATOM 492 H2 TIP3 676 2.702 12.400 -8.501 0.417 1.000 -ATOM 493 OH2 TIP3 679 -12.514 9.742 11.971 -0.834 1.520 -ATOM 494 H1 TIP3 679 -11.884 9.155 11.492 0.417 1.000 -ATOM 495 H2 TIP3 679 -13.222 9.096 12.064 0.417 1.000 -ATOM 496 OH2 TIP3 682 -12.565 11.116 0.100 -0.834 1.520 -ATOM 497 H1 TIP3 682 -13.422 11.264 -0.334 0.417 1.000 -ATOM 498 H2 TIP3 682 -12.686 11.542 1.014 0.417 1.000 -ATOM 499 OH2 TIP3 684 -17.843 16.500 5.355 -0.834 1.520 -ATOM 500 H1 TIP3 684 -17.109 16.045 4.935 0.417 1.000 -ATOM 501 H2 TIP3 684 -17.597 17.436 5.199 0.417 1.000 -ATOM 502 OH2 TIP3 686 -10.854 18.558 18.491 -0.834 1.520 -ATOM 503 H1 TIP3 686 -10.109 17.964 18.496 0.417 1.000 -ATOM 504 H2 TIP3 686 -10.518 19.356 18.944 0.417 1.000 -ATOM 505 OH2 TIP3 699 -8.563 13.795 -0.051 -0.834 1.520 -ATOM 506 H1 TIP3 699 -9.108 13.629 -0.834 0.417 1.000 -ATOM 507 H2 TIP3 699 -9.124 13.721 0.759 0.417 1.000 -ATOM 508 OH2 TIP3 700 -8.050 15.322 14.900 -0.834 1.520 -ATOM 509 H1 TIP3 700 -7.928 14.397 14.787 0.417 1.000 -ATOM 510 H2 TIP3 700 -7.182 15.651 15.219 0.417 1.000 -ATOM 511 OH2 TIP3 701 -13.200 17.116 2.652 -0.834 1.520 -ATOM 512 H1 TIP3 701 -13.097 16.138 2.827 0.417 1.000 -ATOM 513 H2 TIP3 701 -13.771 17.169 1.908 0.417 1.000 -ATOM 514 OH2 TIP3 703 -1.363 15.088 5.879 -0.834 1.520 -ATOM 515 H1 TIP3 703 -1.313 15.420 4.981 0.417 1.000 -ATOM 516 H2 TIP3 703 -1.991 15.716 6.359 0.417 1.000 -ATOM 517 OH2 TIP3 707 -13.869 15.299 19.590 -0.834 1.520 -ATOM 518 H1 TIP3 707 -14.501 15.415 18.908 0.417 1.000 -ATOM 519 H2 TIP3 707 -13.638 14.387 19.519 0.417 1.000 -ATOM 520 OH2 TIP3 708 -3.990 17.713 11.203 -0.834 1.520 -ATOM 521 H1 TIP3 708 -3.036 17.525 10.948 0.417 1.000 -ATOM 522 H2 TIP3 708 -4.293 18.267 10.432 0.417 1.000 -ATOM 523 OH2 TIP3 710 -13.960 16.974 14.283 -0.834 1.520 -ATOM 524 H1 TIP3 710 -13.562 17.353 15.061 0.417 1.000 -ATOM 525 H2 TIP3 710 -13.197 16.869 13.683 0.417 1.000 -ATOM 526 OH2 TIP3 721 -16.049 12.492 0.099 -0.834 1.520 -ATOM 527 H1 TIP3 721 -16.045 11.770 -0.607 0.417 1.000 -ATOM 528 H2 TIP3 721 -16.883 12.282 0.546 0.417 1.000 -ATOM 529 OH2 TIP3 737 -13.666 12.822 10.600 -0.834 1.520 -ATOM 530 H1 TIP3 737 -12.807 12.842 11.097 0.417 1.000 -ATOM 531 H2 TIP3 737 -13.370 12.523 9.704 0.417 1.000 -ATOM 532 OH2 TIP3 738 -15.938 13.760 -14.358 -0.834 1.520 -ATOM 533 H1 TIP3 738 -16.316 14.308 -13.637 0.417 1.000 -ATOM 534 H2 TIP3 738 -16.706 13.253 -14.618 0.417 1.000 -ATOM 535 OH2 TIP3 741 -8.387 14.832 -4.054 -0.834 1.520 -ATOM 536 H1 TIP3 741 -8.331 13.875 -3.795 0.417 1.000 -ATOM 537 H2 TIP3 741 -9.152 15.107 -3.614 0.417 1.000 -ATOM 538 OH2 TIP3 747 -0.600 19.416 11.031 -0.834 1.520 -ATOM 539 H1 TIP3 747 -0.618 18.448 11.348 0.417 1.000 -ATOM 540 H2 TIP3 747 -0.105 19.358 10.220 0.417 1.000 -ATOM 541 OH2 TIP3 759 -13.397 17.068 -6.343 -0.834 1.520 -ATOM 542 H1 TIP3 759 -13.152 16.670 -5.550 0.417 1.000 -ATOM 543 H2 TIP3 759 -13.286 18.036 -6.135 0.417 1.000 -ATOM 544 OH2 TIP3 778 -18.235 19.532 -12.241 -0.834 1.520 -ATOM 545 H1 TIP3 778 -19.239 19.599 -12.280 0.417 1.000 -ATOM 546 H2 TIP3 778 -18.064 18.964 -11.458 0.417 1.000 -ATOM 547 OH2 TIP3 787 0.759 17.628 5.956 -0.834 1.520 -ATOM 548 H1 TIP3 787 0.104 17.147 5.343 0.417 1.000 -ATOM 549 H2 TIP3 787 1.278 18.219 5.409 0.417 1.000 -ATOM 550 OH2 TIP3 822 -19.253 -15.531 -9.421 -0.834 1.520 -ATOM 551 H1 TIP3 822 -19.574 -16.432 -9.661 0.417 1.000 -ATOM 552 H2 TIP3 822 -18.495 -15.415 -10.010 0.417 1.000 -ATOM 553 OH2 TIP3 849 -14.644 -14.612 -3.954 -0.834 1.520 -ATOM 554 H1 TIP3 849 -14.116 -13.936 -3.539 0.417 1.000 -ATOM 555 H2 TIP3 849 -14.408 -15.378 -3.463 0.417 1.000 -ATOM 556 OH2 TIP3 852 -18.111 -14.415 9.541 -0.834 1.520 -ATOM 557 H1 TIP3 852 -18.062 -13.498 9.263 0.417 1.000 -ATOM 558 H2 TIP3 852 -18.985 -14.688 9.145 0.417 1.000 -ATOM 559 OH2 TIP3 867 -9.296 -13.279 -4.117 -0.834 1.520 -ATOM 560 H1 TIP3 867 -9.099 -13.088 -3.164 0.417 1.000 -ATOM 561 H2 TIP3 867 -9.873 -14.028 -4.002 0.417 1.000 -ATOM 562 OH2 TIP3 887 6.897 -7.906 -19.321 -0.834 1.520 -ATOM 563 H1 TIP3 887 6.514 -8.720 -19.696 0.417 1.000 -ATOM 564 H2 TIP3 887 7.875 -8.073 -19.447 0.417 1.000 -ATOM 565 OH2 TIP3 889 -17.811 -18.294 4.878 -0.834 1.520 -ATOM 566 H1 TIP3 889 -18.536 -18.909 4.710 0.417 1.000 -ATOM 567 H2 TIP3 889 -17.875 -18.195 5.900 0.417 1.000 -ATOM 568 OH2 TIP3 891 -8.237 -0.386 -10.042 -0.834 1.520 -ATOM 569 H1 TIP3 891 -7.721 0.405 -10.134 0.417 1.000 -ATOM 570 H2 TIP3 891 -7.565 -1.095 -10.099 0.417 1.000 -ATOM 571 OH2 TIP3 894 -12.136 -9.397 8.950 -0.834 1.520 -ATOM 572 H1 TIP3 894 -12.531 -8.725 9.507 0.417 1.000 -ATOM 573 H2 TIP3 894 -12.846 -10.008 8.783 0.417 1.000 -ATOM 574 OH2 TIP3 895 -15.353 -13.368 17.534 -0.834 1.520 -ATOM 575 H1 TIP3 895 -16.158 -13.824 17.198 0.417 1.000 -ATOM 576 H2 TIP3 895 -15.370 -12.514 17.090 0.417 1.000 -ATOM 577 OH2 TIP3 897 -9.107 -11.584 10.315 -0.834 1.520 -ATOM 578 H1 TIP3 897 -9.261 -12.213 9.538 0.417 1.000 -ATOM 579 H2 TIP3 897 -9.639 -12.029 10.974 0.417 1.000 -ATOM 580 OH2 TIP3 906 -12.973 -17.776 -6.589 -0.834 1.520 -ATOM 581 H1 TIP3 906 -13.815 -17.199 -6.465 0.417 1.000 -ATOM 582 H2 TIP3 906 -13.308 -18.597 -6.258 0.417 1.000 -ATOM 583 OH2 TIP3 907 -2.188 -14.229 -14.124 -0.834 1.520 -ATOM 584 H1 TIP3 907 -2.780 -13.684 -14.620 0.417 1.000 -ATOM 585 H2 TIP3 907 -2.449 -15.131 -14.331 0.417 1.000 -ATOM 586 OH2 TIP3 908 -15.972 -12.301 -7.224 -0.834 1.520 -ATOM 587 H1 TIP3 908 -15.423 -12.807 -7.862 0.417 1.000 -ATOM 588 H2 TIP3 908 -16.829 -12.680 -7.456 0.417 1.000 -ATOM 589 OH2 TIP3 911 -8.951 -10.120 6.882 -0.834 1.520 -ATOM 590 H1 TIP3 911 -8.605 -10.519 7.655 0.417 1.000 -ATOM 591 H2 TIP3 911 -8.095 -9.905 6.383 0.417 1.000 -ATOM 592 OH2 TIP3 912 2.133 -17.114 -0.157 -0.834 1.520 -ATOM 593 H1 TIP3 912 2.929 -17.404 0.262 0.417 1.000 -ATOM 594 H2 TIP3 912 1.486 -17.172 0.561 0.417 1.000 -ATOM 595 OH2 TIP3 916 0.975 -12.700 10.576 -0.834 1.520 -ATOM 596 H1 TIP3 916 1.781 -13.270 10.419 0.417 1.000 -ATOM 597 H2 TIP3 916 1.289 -12.181 11.305 0.417 1.000 -ATOM 598 OH2 TIP3 924 -7.507 -10.549 -17.785 -0.834 1.520 -ATOM 599 H1 TIP3 924 -7.092 -10.160 -17.013 0.417 1.000 -ATOM 600 H2 TIP3 924 -7.018 -11.359 -17.961 0.417 1.000 -ATOM 601 OH2 TIP3 927 -8.823 -10.390 -12.072 -0.834 1.520 -ATOM 602 H1 TIP3 927 -9.557 -9.828 -11.947 0.417 1.000 -ATOM 603 H2 TIP3 927 -9.134 -11.212 -11.702 0.417 1.000 -ATOM 604 OH2 TIP3 928 -18.162 -16.794 -4.602 -0.834 1.520 -ATOM 605 H1 TIP3 928 -17.799 -15.919 -4.455 0.417 1.000 -ATOM 606 H2 TIP3 928 -17.883 -17.001 -5.578 0.417 1.000 -ATOM 607 OH2 TIP3 929 -3.867 -7.407 -3.092 -0.834 1.520 -ATOM 608 H1 TIP3 929 -4.648 -7.231 -3.609 0.417 1.000 -ATOM 609 H2 TIP3 929 -4.113 -8.124 -2.489 0.417 1.000 -ATOM 610 OH2 TIP3 930 -18.101 -8.376 -2.107 -0.834 1.520 -ATOM 611 H1 TIP3 930 -18.861 -8.322 -2.695 0.417 1.000 -ATOM 612 H2 TIP3 930 -17.742 -7.465 -2.145 0.417 1.000 -ATOM 613 OH2 TIP3 932 -10.780 -6.096 -2.660 -0.834 1.520 -ATOM 614 H1 TIP3 932 -9.938 -5.802 -2.262 0.417 1.000 -ATOM 615 H2 TIP3 932 -10.606 -7.001 -2.898 0.417 1.000 -ATOM 616 OH2 TIP3 933 -16.542 -10.132 2.419 -0.834 1.520 -ATOM 617 H1 TIP3 933 -15.602 -9.984 2.654 0.417 1.000 -ATOM 618 H2 TIP3 933 -16.552 -9.811 1.495 0.417 1.000 -ATOM 619 OH2 TIP3 936 -17.076 -15.857 6.959 -0.834 1.520 -ATOM 620 H1 TIP3 936 -16.442 -15.454 6.376 0.417 1.000 -ATOM 621 H2 TIP3 936 -17.335 -15.170 7.592 0.417 1.000 -ATOM 622 OH2 TIP3 940 -19.250 -17.062 13.461 -0.834 1.520 -ATOM 623 H1 TIP3 940 -19.256 -17.491 14.315 0.417 1.000 -ATOM 624 H2 TIP3 940 -19.609 -16.191 13.743 0.417 1.000 -ATOM 625 OH2 TIP3 941 -9.333 -7.566 15.503 -0.834 1.520 -ATOM 626 H1 TIP3 941 -10.126 -7.052 15.227 0.417 1.000 -ATOM 627 H2 TIP3 941 -9.061 -7.055 16.214 0.417 1.000 -ATOM 628 OH2 TIP3 942 -1.559 -3.866 -16.160 -0.834 1.520 -ATOM 629 H1 TIP3 942 -1.390 -3.512 -17.032 0.417 1.000 -ATOM 630 H2 TIP3 942 -0.887 -4.593 -16.039 0.417 1.000 -ATOM 631 OH2 TIP3 947 -4.570 -16.726 -12.593 -0.834 1.520 -ATOM 632 H1 TIP3 947 -4.542 -17.259 -11.843 0.417 1.000 -ATOM 633 H2 TIP3 947 -4.067 -17.260 -13.230 0.417 1.000 -ATOM 634 OH2 TIP3 948 -6.866 -15.465 -13.441 -0.834 1.520 -ATOM 635 H1 TIP3 948 -7.566 -16.068 -13.062 0.417 1.000 -ATOM 636 H2 TIP3 948 -6.041 -15.838 -13.035 0.417 1.000 -ATOM 637 OH2 TIP3 949 -14.292 -15.574 4.418 -0.834 1.520 -ATOM 638 H1 TIP3 949 -13.998 -15.297 5.300 0.417 1.000 -ATOM 639 H2 TIP3 949 -14.843 -14.832 4.144 0.417 1.000 -ATOM 640 OH2 TIP3 950 -3.679 -0.505 -4.130 -0.834 1.520 -ATOM 641 H1 TIP3 950 -3.040 -0.841 -4.808 0.417 1.000 -ATOM 642 H2 TIP3 950 -3.146 0.132 -3.612 0.417 1.000 -ATOM 643 OH2 TIP3 951 -9.705 -5.934 -5.680 -0.834 1.520 -ATOM 644 H1 TIP3 951 -9.408 -5.933 -4.774 0.417 1.000 -ATOM 645 H2 TIP3 951 -10.678 -5.774 -5.650 0.417 1.000 -ATOM 646 OH2 TIP3 952 -3.992 -2.203 -8.590 -0.834 1.520 -ATOM 647 H1 TIP3 952 -3.305 -2.298 -9.316 0.417 1.000 -ATOM 648 H2 TIP3 952 -4.814 -2.479 -9.058 0.417 1.000 -ATOM 649 OH2 TIP3 953 -4.602 -18.448 -1.754 -0.834 1.520 -ATOM 650 H1 TIP3 953 -3.984 -18.049 -2.434 0.417 1.000 -ATOM 651 H2 TIP3 953 -5.426 -17.989 -1.891 0.417 1.000 -ATOM 652 OH2 TIP3 964 -12.170 -0.249 -18.005 -0.834 1.520 -ATOM 653 H1 TIP3 964 -12.743 -0.130 -18.753 0.417 1.000 -ATOM 654 H2 TIP3 964 -11.671 -1.077 -18.115 0.417 1.000 -ATOM 655 OH2 TIP3 970 -6.337 6.432 -13.471 -0.834 1.520 -ATOM 656 H1 TIP3 970 -6.149 6.261 -12.518 0.417 1.000 -ATOM 657 H2 TIP3 970 -6.645 7.375 -13.468 0.417 1.000 -ATOM 658 OH2 TIP3 972 -4.973 -8.751 -10.194 -0.834 1.520 -ATOM 659 H1 TIP3 972 -4.364 -9.047 -10.896 0.417 1.000 -ATOM 660 H2 TIP3 972 -5.071 -7.806 -10.455 0.417 1.000 -ATOM 661 OH2 TIP3 973 -0.470 -0.187 4.413 -0.834 1.520 -ATOM 662 H1 TIP3 973 0.399 0.251 4.331 0.417 1.000 -ATOM 663 H2 TIP3 973 -0.245 -1.124 4.418 0.417 1.000 -ATOM 664 OH2 TIP3 974 -9.726 -8.552 0.959 -0.834 1.520 -ATOM 665 H1 TIP3 974 -9.107 -8.927 1.654 0.417 1.000 -ATOM 666 H2 TIP3 974 -9.656 -9.109 0.238 0.417 1.000 -ATOM 667 OH2 TIP3 975 -8.527 -13.925 1.702 -0.834 1.520 -ATOM 668 H1 TIP3 975 -7.954 -14.333 2.431 0.417 1.000 -ATOM 669 H2 TIP3 975 -9.446 -14.061 2.100 0.417 1.000 -ATOM 670 OH2 TIP3 977 -11.682 -4.716 5.150 -0.834 1.520 -ATOM 671 H1 TIP3 977 -11.010 -4.921 5.736 0.417 1.000 -ATOM 672 H2 TIP3 977 -11.264 -4.047 4.564 0.417 1.000 -ATOM 673 OH2 TIP3 985 10.868 8.734 -17.269 -0.834 1.520 -ATOM 674 H1 TIP3 985 10.991 9.422 -17.981 0.417 1.000 -ATOM 675 H2 TIP3 985 11.515 8.043 -17.516 0.417 1.000 -ATOM 676 OH2 TIP3 988 -6.088 6.462 -17.714 -0.834 1.520 -ATOM 677 H1 TIP3 988 -6.333 7.388 -17.461 0.417 1.000 -ATOM 678 H2 TIP3 988 -6.512 5.916 -17.000 0.417 1.000 -ATOM 679 OH2 TIP3 989 -8.835 12.860 -18.664 -0.834 1.520 -ATOM 680 H1 TIP3 989 -8.617 12.035 -19.026 0.417 1.000 -ATOM 681 H2 TIP3 989 -9.094 12.669 -17.759 0.417 1.000 -ATOM 682 OH2 TIP3 990 4.449 16.400 -12.116 -0.834 1.520 -ATOM 683 H1 TIP3 990 5.434 16.551 -12.143 0.417 1.000 -ATOM 684 H2 TIP3 990 4.142 16.332 -13.058 0.417 1.000 -ATOM 685 OH2 TIP3 991 -5.665 14.997 -4.341 -0.834 1.520 -ATOM 686 H1 TIP3 991 -6.636 14.868 -4.261 0.417 1.000 -ATOM 687 H2 TIP3 991 -5.590 15.230 -5.279 0.417 1.000 -ATOM 688 OH2 TIP3 992 1.008 -16.730 -4.827 -0.834 1.520 -ATOM 689 H1 TIP3 992 0.390 -15.964 -4.916 0.417 1.000 -ATOM 690 H2 TIP3 992 1.768 -16.408 -5.251 0.417 1.000 -ATOM 691 OH2 TIP3 994 -2.690 -7.275 10.317 -0.834 1.520 -ATOM 692 H1 TIP3 994 -3.519 -6.839 10.591 0.417 1.000 -ATOM 693 H2 TIP3 994 -2.943 -8.144 9.939 0.417 1.000 -ATOM 694 OH2 TIP3 995 -13.200 -3.341 -2.816 -0.834 1.520 -ATOM 695 H1 TIP3 995 -12.857 -3.351 -3.673 0.417 1.000 -ATOM 696 H2 TIP3 995 -12.891 -2.498 -2.432 0.417 1.000 -ATOM 697 OH2 TIP3 996 -14.220 -2.484 4.024 -0.834 1.520 -ATOM 698 H1 TIP3 996 -14.894 -2.674 3.341 0.417 1.000 -ATOM 699 H2 TIP3 996 -14.250 -1.515 4.094 0.417 1.000 -ATOM 700 OH2 TIP3 998 4.714 1.300 8.954 -0.834 1.520 -ATOM 701 H1 TIP3 998 4.446 1.591 8.048 0.417 1.000 -ATOM 702 H2 TIP3 998 5.209 2.035 9.317 0.417 1.000 -ATOM 703 OH2 TIP3 1007 -5.397 -1.890 -16.787 -0.834 1.520 -ATOM 704 H1 TIP3 1007 -5.725 -2.345 -16.037 0.417 1.000 -ATOM 705 H2 TIP3 1007 -6.158 -1.394 -17.122 0.417 1.000 -ATOM 706 OH2 TIP3 1011 -15.247 0.139 -14.525 -0.834 1.520 -ATOM 707 H1 TIP3 1011 -15.231 1.053 -14.154 0.417 1.000 -ATOM 708 H2 TIP3 1011 -14.337 0.031 -14.799 0.417 1.000 -ATOM 709 OH2 TIP3 1012 -10.750 2.403 8.404 -0.834 1.520 -ATOM 710 H1 TIP3 1012 -11.576 2.852 8.305 0.417 1.000 -ATOM 711 H2 TIP3 1012 -10.857 1.926 9.253 0.417 1.000 -ATOM 712 OH2 TIP3 1013 -18.385 -2.645 -5.645 -0.834 1.520 -ATOM 713 H1 TIP3 1013 -18.775 -3.520 -5.580 0.417 1.000 -ATOM 714 H2 TIP3 1013 -19.141 -2.016 -5.949 0.417 1.000 -ATOM 715 OH2 TIP3 1014 -8.913 -4.213 0.150 -0.834 1.520 -ATOM 716 H1 TIP3 1014 -8.779 -5.123 -0.302 0.417 1.000 -ATOM 717 H2 TIP3 1014 -8.064 -4.060 0.591 0.417 1.000 -ATOM 718 OH2 TIP3 1015 2.578 3.665 3.229 -0.834 1.520 -ATOM 719 H1 TIP3 1015 3.396 3.499 2.734 0.417 1.000 -ATOM 720 H2 TIP3 1015 1.920 3.775 2.561 0.417 1.000 -ATOM 721 OH2 TIP3 1017 -10.216 -1.940 -3.561 -0.834 1.520 -ATOM 722 H1 TIP3 1017 -9.424 -1.834 -4.078 0.417 1.000 -ATOM 723 H2 TIP3 1017 -10.059 -1.416 -2.766 0.417 1.000 -ATOM 724 OH2 TIP3 1018 -17.631 -0.380 -9.162 -0.834 1.520 -ATOM 725 H1 TIP3 1018 -17.949 -0.575 -8.299 0.417 1.000 -ATOM 726 H2 TIP3 1018 -16.653 -0.283 -9.077 0.417 1.000 -ATOM 727 OH2 TIP3 1020 -7.801 -8.646 11.965 -0.834 1.520 -ATOM 728 H1 TIP3 1020 -7.720 -7.730 12.149 0.417 1.000 -ATOM 729 H2 TIP3 1020 -8.615 -8.646 11.486 0.417 1.000 -ATOM 730 OH2 TIP3 1023 -13.013 5.343 12.803 -0.834 1.520 -ATOM 731 H1 TIP3 1023 -13.938 5.093 13.100 0.417 1.000 -ATOM 732 H2 TIP3 1023 -12.532 4.995 13.508 0.417 1.000 -ATOM 733 OH2 TIP3 1027 -4.842 -4.782 -18.547 -0.834 1.520 -ATOM 734 H1 TIP3 1027 -4.714 -3.877 -18.840 0.417 1.000 -ATOM 735 H2 TIP3 1027 -5.825 -4.781 -18.413 0.417 1.000 -ATOM 736 OH2 TIP3 1032 -12.071 -10.399 -15.678 -0.834 1.520 -ATOM 737 H1 TIP3 1032 -12.636 -9.660 -16.088 0.417 1.000 -ATOM 738 H2 TIP3 1032 -12.682 -10.989 -15.180 0.417 1.000 -ATOM 739 OH2 TIP3 1033 -9.791 6.935 -18.027 -0.834 1.520 -ATOM 740 H1 TIP3 1033 -10.603 7.288 -18.480 0.417 1.000 -ATOM 741 H2 TIP3 1033 -9.477 7.672 -17.376 0.417 1.000 -ATOM 742 OH2 TIP3 1039 -10.941 13.382 11.598 -0.834 1.520 -ATOM 743 H1 TIP3 1039 -10.702 12.548 11.121 0.417 1.000 -ATOM 744 H2 TIP3 1039 -10.814 13.114 12.523 0.417 1.000 -ATOM 745 OH2 TIP3 1040 -12.875 6.008 -3.795 -0.834 1.520 -ATOM 746 H1 TIP3 1040 -12.695 6.912 -4.006 0.417 1.000 -ATOM 747 H2 TIP3 1040 -13.715 5.889 -4.266 0.417 1.000 -ATOM 748 OH2 TIP3 1053 -8.415 3.121 -0.545 -0.834 1.520 -ATOM 749 H1 TIP3 1053 -7.587 3.205 -0.018 0.417 1.000 -ATOM 750 H2 TIP3 1053 -9.184 3.266 0.095 0.417 1.000 -ATOM 751 OH2 TIP3 1054 1.033 7.960 -2.407 -0.834 1.520 -ATOM 752 H1 TIP3 1054 1.033 8.956 -2.379 0.417 1.000 -ATOM 753 H2 TIP3 1054 1.925 7.777 -2.825 0.417 1.000 -ATOM 754 OH2 TIP3 1055 -11.543 11.011 -19.140 -0.834 1.520 -ATOM 755 H1 TIP3 1055 -11.756 11.664 -18.486 0.417 1.000 -ATOM 756 H2 TIP3 1055 -12.191 10.305 -18.933 0.417 1.000 -ATOM 757 OH2 TIP3 1056 -0.599 16.861 3.848 -0.834 1.520 -ATOM 758 H1 TIP3 1056 -0.612 17.833 3.651 0.417 1.000 -ATOM 759 H2 TIP3 1056 -1.025 16.454 3.026 0.417 1.000 -ATOM 760 OH2 TIP3 1057 -10.077 18.910 2.513 -0.834 1.520 -ATOM 761 H1 TIP3 1057 -10.245 18.506 3.457 0.417 1.000 -ATOM 762 H2 TIP3 1057 -10.853 18.717 1.948 0.417 1.000 -ATOM 763 OH2 TIP3 1058 -13.751 7.735 -6.332 -0.834 1.520 -ATOM 764 H1 TIP3 1058 -13.537 8.358 -7.002 0.417 1.000 -ATOM 765 H2 TIP3 1058 -12.910 7.243 -6.152 0.417 1.000 -ATOM 766 OH2 TIP3 1059 -6.245 5.794 -10.480 -0.834 1.520 -ATOM 767 H1 TIP3 1059 -7.174 5.669 -10.233 0.417 1.000 -ATOM 768 H2 TIP3 1059 -5.852 5.751 -9.626 0.417 1.000 -ATOM 769 OH2 TIP3 1060 -8.580 0.031 14.330 -0.834 1.520 -ATOM 770 H1 TIP3 1060 -9.257 0.735 14.495 0.417 1.000 -ATOM 771 H2 TIP3 1060 -8.817 -0.665 14.947 0.417 1.000 -ATOM 772 OH2 TIP3 1061 -16.604 -0.006 16.503 -0.834 1.520 -ATOM 773 H1 TIP3 1061 -16.646 -0.905 16.141 0.417 1.000 -ATOM 774 H2 TIP3 1061 -16.475 0.554 15.622 0.417 1.000 -ATOM 775 OH2 TIP3 1062 -4.502 6.311 17.716 -0.834 1.520 -ATOM 776 H1 TIP3 1062 -4.891 7.146 17.683 0.417 1.000 -ATOM 777 H2 TIP3 1062 -4.551 5.975 16.834 0.417 1.000 -ATOM 778 OH2 TIP3 1063 -6.374 10.844 16.012 -0.834 1.520 -ATOM 779 H1 TIP3 1063 -7.039 10.601 16.641 0.417 1.000 -ATOM 780 H2 TIP3 1063 -5.591 10.739 16.544 0.417 1.000 -ATOM 781 OH2 TIP3 1065 -3.665 18.240 18.854 -0.834 1.520 -ATOM 782 H1 TIP3 1065 -3.807 17.407 19.375 0.417 1.000 -ATOM 783 H2 TIP3 1065 -4.496 18.528 18.513 0.417 1.000 -ATOM 784 OH2 TIP3 1066 -16.259 13.268 11.520 -0.834 1.520 -ATOM 785 H1 TIP3 1066 -15.303 13.107 11.390 0.417 1.000 -ATOM 786 H2 TIP3 1066 -16.330 13.920 12.193 0.417 1.000 -ATOM 787 OH2 TIP3 1076 -17.480 10.670 -13.743 -0.834 1.520 -ATOM 788 H1 TIP3 1076 -17.328 9.751 -13.563 0.417 1.000 -ATOM 789 H2 TIP3 1076 -17.584 11.150 -12.855 0.417 1.000 -ATOM 790 OH2 TIP3 1079 3.653 5.472 10.275 -0.834 1.520 -ATOM 791 H1 TIP3 1079 2.816 5.942 10.562 0.417 1.000 -ATOM 792 H2 TIP3 1079 3.255 4.843 9.700 0.417 1.000 -ATOM 793 OH2 TIP3 1080 -14.020 7.548 3.471 -0.834 1.520 -ATOM 794 H1 TIP3 1080 -13.723 8.300 3.032 0.417 1.000 -ATOM 795 H2 TIP3 1080 -14.003 7.744 4.375 0.417 1.000 -ATOM 796 OH2 TIP3 1081 -12.309 14.925 7.136 -0.834 1.520 -ATOM 797 H1 TIP3 1081 -11.552 14.853 6.499 0.417 1.000 -ATOM 798 H2 TIP3 1081 -11.810 15.003 7.987 0.417 1.000 -ATOM 799 OH2 TIP3 1082 -0.176 3.862 17.888 -0.834 1.520 -ATOM 800 H1 TIP3 1082 0.398 3.217 17.384 0.417 1.000 -ATOM 801 H2 TIP3 1082 0.523 4.255 18.430 0.417 1.000 -ATOM 802 OH2 TIP3 1096 -0.168 9.792 -14.480 -0.834 1.520 -ATOM 803 H1 TIP3 1096 0.273 10.362 -13.807 0.417 1.000 -ATOM 804 H2 TIP3 1096 -0.633 9.183 -13.951 0.417 1.000 -ATOM 805 OH2 TIP3 1097 -10.265 12.833 14.356 -0.834 1.520 -ATOM 806 H1 TIP3 1097 -9.333 12.961 14.414 0.417 1.000 -ATOM 807 H2 TIP3 1097 -10.520 12.294 15.174 0.417 1.000 -ATOM 808 OH2 TIP3 1098 -16.741 9.755 -9.522 -0.834 1.520 -ATOM 809 H1 TIP3 1098 -16.623 10.314 -10.277 0.417 1.000 -ATOM 810 H2 TIP3 1098 -17.502 9.224 -9.857 0.417 1.000 -ATOM 811 OH2 TIP3 1099 -7.151 15.375 6.653 -0.834 1.520 -ATOM 812 H1 TIP3 1099 -6.424 14.867 6.350 0.417 1.000 -ATOM 813 H2 TIP3 1099 -7.757 14.708 7.004 0.417 1.000 -ATOM 814 OH2 TIP3 1100 -13.262 3.712 6.178 -0.834 1.520 -ATOM 815 H1 TIP3 1100 -12.917 4.220 5.441 0.417 1.000 -ATOM 816 H2 TIP3 1100 -14.093 3.381 5.843 0.417 1.000 -ATOM 817 OH2 TIP3 1101 -8.171 17.598 1.309 -0.834 1.520 -ATOM 818 H1 TIP3 1101 -8.803 17.629 2.074 0.417 1.000 -ATOM 819 H2 TIP3 1101 -8.386 18.399 0.840 0.417 1.000 -ATOM 820 OH2 TIP3 1104 -18.622 11.718 11.127 -0.834 1.520 -ATOM 821 H1 TIP3 1104 -17.850 12.172 11.554 0.417 1.000 -ATOM 822 H2 TIP3 1104 -18.943 11.061 11.774 0.417 1.000 -ATOM 823 OH2 TIP3 1105 -10.120 19.166 15.174 -0.834 1.520 -ATOM 824 H1 TIP3 1105 -10.720 18.846 15.891 0.417 1.000 -ATOM 825 H2 TIP3 1105 -9.266 19.097 15.577 0.417 1.000 -ATOM 826 OH2 TIP3 1107 -13.117 6.827 15.132 -0.834 1.520 -ATOM 827 H1 TIP3 1107 -13.335 6.501 14.198 0.417 1.000 -ATOM 828 H2 TIP3 1107 -13.990 6.720 15.589 0.417 1.000 -ATOM 829 OH2 TIP3 1112 -13.297 6.593 -15.152 -0.834 1.520 -ATOM 830 H1 TIP3 1112 -12.593 7.207 -15.239 0.417 1.000 -ATOM 831 H2 TIP3 1112 -13.284 6.068 -15.987 0.417 1.000 -ATOM 832 OH2 TIP3 1115 1.371 -2.115 -14.125 -0.834 1.520 -ATOM 833 H1 TIP3 1115 0.731 -2.810 -13.799 0.417 1.000 -ATOM 834 H2 TIP3 1115 2.157 -2.229 -13.621 0.417 1.000 -ATOM 835 OH2 TIP3 1117 1.052 -5.510 -15.708 -0.834 1.520 -ATOM 836 H1 TIP3 1117 0.664 -5.975 -14.991 0.417 1.000 -ATOM 837 H2 TIP3 1117 1.791 -4.994 -15.253 0.417 1.000 -ATOM 838 OH2 TIP3 1118 -8.935 8.816 4.528 -0.834 1.520 -ATOM 839 H1 TIP3 1118 -9.687 8.525 5.001 0.417 1.000 -ATOM 840 H2 TIP3 1118 -8.280 8.079 4.603 0.417 1.000 -ATOM 841 OH2 TIP3 1120 -11.893 9.357 5.179 -0.834 1.520 -ATOM 842 H1 TIP3 1120 -12.419 9.729 5.895 0.417 1.000 -ATOM 843 H2 TIP3 1120 -11.572 8.414 5.546 0.417 1.000 -ATOM 844 OH2 TIP3 1121 -8.926 17.222 7.267 -0.834 1.520 -ATOM 845 H1 TIP3 1121 -8.769 17.549 8.221 0.417 1.000 -ATOM 846 H2 TIP3 1121 -8.247 16.451 7.072 0.417 1.000 -ATOM 847 OH2 TIP3 1122 -18.269 11.479 1.005 -0.834 1.520 -ATOM 848 H1 TIP3 1122 -18.118 10.862 0.348 0.417 1.000 -ATOM 849 H2 TIP3 1122 -18.239 10.976 1.853 0.417 1.000 -ATOM 850 OH2 TIP3 1145 2.643 9.951 12.565 -0.834 1.520 -ATOM 851 H1 TIP3 1145 1.799 9.493 12.514 0.417 1.000 -ATOM 852 H2 TIP3 1145 3.225 9.166 12.828 0.417 1.000 -ATOM 853 OH2 TIP3 1147 -5.856 7.712 15.317 -0.834 1.520 -ATOM 854 H1 TIP3 1147 -6.487 7.643 16.062 0.417 1.000 -ATOM 855 H2 TIP3 1147 -6.061 7.011 14.727 0.417 1.000 -ATOM 856 OH2 TIP3 1148 -12.910 2.613 17.614 -0.834 1.520 -ATOM 857 H1 TIP3 1148 -13.894 2.435 17.632 0.417 1.000 -ATOM 858 H2 TIP3 1148 -12.600 1.991 18.307 0.417 1.000 -ATOM 859 OH2 TIP3 1158 -12.889 17.051 -3.520 -0.834 1.520 -ATOM 860 H1 TIP3 1158 -13.741 16.733 -3.249 0.417 1.000 -ATOM 861 H2 TIP3 1158 -12.803 17.960 -3.232 0.417 1.000 -ATOM 862 OH2 TIP3 1164 -4.220 1.074 14.906 -0.834 1.520 -ATOM 863 H1 TIP3 1164 -4.321 0.862 15.797 0.417 1.000 -ATOM 864 H2 TIP3 1164 -3.250 1.309 14.791 0.417 1.000 -ATOM 865 OH2 TIP3 1178 -11.767 18.772 -16.664 -0.834 1.520 -ATOM 866 H1 TIP3 1178 -11.201 18.275 -16.113 0.417 1.000 -ATOM 867 H2 TIP3 1178 -12.082 19.471 -16.044 0.417 1.000 -ATOM 868 OH2 TIP3 1182 -15.666 19.078 8.966 -0.834 1.520 -ATOM 869 H1 TIP3 1182 -14.895 19.164 8.467 0.417 1.000 -ATOM 870 H2 TIP3 1182 -16.080 18.301 8.580 0.417 1.000 -ATOM 871 OH2 TIP3 1183 -13.849 16.124 -8.843 -0.834 1.520 -ATOM 872 H1 TIP3 1183 -13.748 16.551 -7.967 0.417 1.000 -ATOM 873 H2 TIP3 1183 -13.363 16.696 -9.489 0.417 1.000 -ATOM 874 OH2 TIP3 1186 -14.056 10.005 17.555 -0.834 1.520 -ATOM 875 H1 TIP3 1186 -14.565 9.721 18.312 0.417 1.000 -ATOM 876 H2 TIP3 1186 -13.501 9.263 17.322 0.417 1.000 -ATOM 877 OH2 TIP3 1188 -10.497 19.714 8.657 -0.834 1.520 -ATOM 878 H1 TIP3 1188 -10.010 19.551 7.781 0.417 1.000 -ATOM 879 H2 TIP3 1188 -9.798 19.991 9.334 0.417 1.000 -ATOM 880 OH2 TIP3 1194 -15.074 18.691 17.583 -0.834 1.520 -ATOM 881 H1 TIP3 1194 -15.526 19.517 17.888 0.417 1.000 -ATOM 882 H2 TIP3 1194 -15.789 18.140 17.209 0.417 1.000 -ATOM 883 OH2 TIP3 1197 -8.107 13.506 -9.559 -0.834 1.520 -ATOM 884 H1 TIP3 1197 -8.901 13.627 -9.090 0.417 1.000 -ATOM 885 H2 TIP3 1197 -7.648 14.369 -9.549 0.417 1.000 -ATOM 886 OH2 TIP3 1200 -13.364 18.508 -10.693 -0.834 1.520 -ATOM 887 H1 TIP3 1200 -14.034 18.183 -11.177 0.417 1.000 -ATOM 888 H2 TIP3 1200 -13.662 19.353 -10.245 0.417 1.000 -ATOM 889 OH2 TIP3 1201 -14.659 9.958 -7.817 -0.834 1.520 -ATOM 890 H1 TIP3 1201 -15.489 9.806 -8.267 0.417 1.000 -ATOM 891 H2 TIP3 1201 -14.243 10.640 -8.384 0.417 1.000 -ATOM 892 OH2 TIP3 1202 -2.734 13.646 0.485 -0.834 1.520 -ATOM 893 H1 TIP3 1202 -2.323 12.967 1.063 0.417 1.000 -ATOM 894 H2 TIP3 1202 -2.479 14.453 0.986 0.417 1.000 -ATOM 895 OH2 TIP3 1272 -11.212 -16.706 4.627 -0.834 1.520 -ATOM 896 H1 TIP3 1272 -12.126 -16.430 4.750 0.417 1.000 -ATOM 897 H2 TIP3 1272 -10.829 -16.091 4.019 0.417 1.000 -ATOM 898 OH2 TIP3 1274 -12.561 -18.940 14.760 -0.834 1.520 -ATOM 899 H1 TIP3 1274 -12.143 -18.084 14.514 0.417 1.000 -ATOM 900 H2 TIP3 1274 -12.552 -19.454 13.976 0.417 1.000 -ATOM 901 OH2 TIP3 1298 -19.148 -15.823 17.753 -0.834 1.520 -ATOM 902 H1 TIP3 1298 -18.276 -16.222 17.993 0.417 1.000 -ATOM 903 H2 TIP3 1298 -19.768 -16.555 17.885 0.417 1.000 -ATOM 904 OH2 TIP3 1301 -16.680 -18.561 -16.307 -0.834 1.520 -ATOM 905 H1 TIP3 1301 -16.982 -18.737 -17.205 0.417 1.000 -ATOM 906 H2 TIP3 1301 -15.782 -18.263 -16.467 0.417 1.000 -ATOM 907 OH2 TIP3 1307 -3.904 -11.595 -9.372 -0.834 1.520 -ATOM 908 H1 TIP3 1307 -3.915 -10.610 -9.285 0.417 1.000 -ATOM 909 H2 TIP3 1307 -3.923 -11.652 -10.379 0.417 1.000 -ATOM 910 OH2 TIP3 1308 -11.622 2.836 -5.926 -0.834 1.520 -ATOM 911 H1 TIP3 1308 -10.873 3.085 -6.522 0.417 1.000 -ATOM 912 H2 TIP3 1308 -12.194 3.652 -5.888 0.417 1.000 -ATOM 913 OH2 TIP3 1309 4.434 -15.226 13.071 -0.834 1.520 -ATOM 914 H1 TIP3 1309 5.249 -15.245 12.565 0.417 1.000 -ATOM 915 H2 TIP3 1309 4.297 -16.185 13.213 0.417 1.000 -ATOM 916 OH2 TIP3 1311 -8.145 -15.478 9.839 -0.834 1.520 -ATOM 917 H1 TIP3 1311 -7.664 -16.276 9.523 0.417 1.000 -ATOM 918 H2 TIP3 1311 -8.668 -15.764 10.567 0.417 1.000 -ATOM 919 OH2 TIP3 1313 -9.574 -0.458 7.064 -0.834 1.520 -ATOM 920 H1 TIP3 1313 -10.532 -0.612 7.335 0.417 1.000 -ATOM 921 H2 TIP3 1313 -9.347 -1.197 6.429 0.417 1.000 -ATOM 922 OH2 TIP3 1326 -1.521 -16.911 -18.622 -0.834 1.520 -ATOM 923 H1 TIP3 1326 -2.464 -16.792 -18.426 0.417 1.000 -ATOM 924 H2 TIP3 1326 -1.398 -17.860 -18.803 0.417 1.000 -ATOM 925 OH2 TIP3 1327 -4.471 -14.018 -11.481 -0.834 1.520 -ATOM 926 H1 TIP3 1327 -3.602 -14.368 -11.188 0.417 1.000 -ATOM 927 H2 TIP3 1327 -4.777 -14.667 -12.133 0.417 1.000 -ATOM 928 OH2 TIP3 1328 -15.461 -17.395 -11.705 -0.834 1.520 -ATOM 929 H1 TIP3 1328 -14.540 -17.298 -11.877 0.417 1.000 -ATOM 930 H2 TIP3 1328 -15.566 -18.384 -11.637 0.417 1.000 -ATOM 931 OH2 TIP3 1329 0.028 -12.920 -10.978 -0.834 1.520 -ATOM 932 H1 TIP3 1329 0.675 -12.285 -10.918 0.417 1.000 -ATOM 933 H2 TIP3 1329 -0.615 -12.427 -11.473 0.417 1.000 -ATOM 934 OH2 TIP3 1330 2.010 -8.972 -1.553 -0.834 1.520 -ATOM 935 H1 TIP3 1330 2.719 -8.939 -2.299 0.417 1.000 -ATOM 936 H2 TIP3 1330 2.590 -8.860 -0.780 0.417 1.000 -ATOM 937 OH2 TIP3 1332 -5.323 -17.212 9.301 -0.834 1.520 -ATOM 938 H1 TIP3 1332 -5.340 -16.370 8.811 0.417 1.000 -ATOM 939 H2 TIP3 1332 -5.459 -17.798 8.553 0.417 1.000 -ATOM 940 OH2 TIP3 1333 -1.650 -13.022 -2.385 -0.834 1.520 -ATOM 941 H1 TIP3 1333 -1.585 -12.709 -3.298 0.417 1.000 -ATOM 942 H2 TIP3 1333 -2.602 -12.841 -2.216 0.417 1.000 -ATOM 943 OH2 TIP3 1344 -11.117 1.581 -14.240 -0.834 1.520 -ATOM 944 H1 TIP3 1344 -11.295 2.232 -13.554 0.417 1.000 -ATOM 945 H2 TIP3 1344 -10.643 2.008 -14.923 0.417 1.000 -ATOM 946 OH2 TIP3 1346 -0.601 -7.636 -18.020 -0.834 1.520 -ATOM 947 H1 TIP3 1346 -0.285 -8.501 -18.402 0.417 1.000 -ATOM 948 H2 TIP3 1346 -0.909 -7.941 -17.154 0.417 1.000 -ATOM 949 OH2 TIP3 1347 4.165 -13.181 -14.225 -0.834 1.520 -ATOM 950 H1 TIP3 1347 3.867 -13.390 -15.127 0.417 1.000 -ATOM 951 H2 TIP3 1347 3.362 -12.823 -13.749 0.417 1.000 -ATOM 952 OH2 TIP3 1349 -7.207 -6.636 -7.832 -0.834 1.520 -ATOM 953 H1 TIP3 1349 -7.213 -6.452 -8.777 0.417 1.000 -ATOM 954 H2 TIP3 1349 -8.005 -6.206 -7.601 0.417 1.000 -ATOM 955 OH2 TIP3 1352 -13.333 -12.453 -10.354 -0.834 1.520 -ATOM 956 H1 TIP3 1352 -13.897 -13.025 -9.704 0.417 1.000 -ATOM 957 H2 TIP3 1352 -12.442 -12.708 -9.954 0.417 1.000 -ATOM 958 OH2 TIP3 1354 -19.818 3.403 1.361 -0.834 1.520 -ATOM 959 H1 TIP3 1354 -19.699 3.585 2.346 0.417 1.000 -ATOM 960 H2 TIP3 1354 -18.999 2.867 1.220 0.417 1.000 -ATOM 961 OH2 TIP3 1355 -8.653 -3.993 3.148 -0.834 1.520 -ATOM 962 H1 TIP3 1355 -8.221 -4.874 3.087 0.417 1.000 -ATOM 963 H2 TIP3 1355 -9.414 -4.077 2.581 0.417 1.000 -ATOM 964 OH2 TIP3 1356 -14.219 -11.030 5.936 -0.834 1.520 -ATOM 965 H1 TIP3 1356 -13.821 -10.823 5.053 0.417 1.000 -ATOM 966 H2 TIP3 1356 -13.529 -11.567 6.363 0.417 1.000 -ATOM 967 OH2 TIP3 1361 -2.743 9.681 13.676 -0.834 1.520 -ATOM 968 H1 TIP3 1361 -2.802 10.631 13.366 0.417 1.000 -ATOM 969 H2 TIP3 1361 -3.648 9.417 13.384 0.417 1.000 -ATOM 970 OH2 TIP3 1367 -9.871 -2.515 -13.450 -0.834 1.520 -ATOM 971 H1 TIP3 1367 -10.769 -2.533 -13.080 0.417 1.000 -ATOM 972 H2 TIP3 1367 -9.892 -3.298 -14.015 0.417 1.000 -ATOM 973 OH2 TIP3 1368 2.412 -12.394 -7.335 -0.834 1.520 -ATOM 974 H1 TIP3 1368 1.852 -12.020 -6.612 0.417 1.000 -ATOM 975 H2 TIP3 1368 1.958 -13.231 -7.507 0.417 1.000 -ATOM 976 OH2 TIP3 1370 -15.346 -14.283 -14.977 -0.834 1.520 -ATOM 977 H1 TIP3 1370 -14.941 -14.874 -15.605 0.417 1.000 -ATOM 978 H2 TIP3 1370 -14.591 -14.215 -14.366 0.417 1.000 -ATOM 979 OH2 TIP3 1371 -7.664 -5.095 -10.650 -0.834 1.520 -ATOM 980 H1 TIP3 1371 -8.151 -4.412 -11.117 0.417 1.000 -ATOM 981 H2 TIP3 1371 -8.432 -5.625 -10.230 0.417 1.000 -ATOM 982 OH2 TIP3 1372 -14.813 -18.301 4.688 -0.834 1.520 -ATOM 983 H1 TIP3 1372 -14.504 -17.407 4.739 0.417 1.000 -ATOM 984 H2 TIP3 1372 -15.831 -18.127 4.767 0.417 1.000 -ATOM 985 OH2 TIP3 1374 -14.512 -5.745 -6.788 -0.834 1.520 -ATOM 986 H1 TIP3 1374 -13.721 -5.874 -6.291 0.417 1.000 -ATOM 987 H2 TIP3 1374 -15.136 -6.009 -6.122 0.417 1.000 -ATOM 988 OH2 TIP3 1376 -7.402 -6.842 3.181 -0.834 1.520 -ATOM 989 H1 TIP3 1376 -7.831 -6.794 4.069 0.417 1.000 -ATOM 990 H2 TIP3 1376 -6.484 -6.977 3.463 0.417 1.000 -ATOM 991 OH2 TIP3 1387 -16.314 -11.647 -16.599 -0.834 1.520 -ATOM 992 H1 TIP3 1387 -15.506 -12.070 -16.916 0.417 1.000 -ATOM 993 H2 TIP3 1387 -16.959 -12.273 -16.803 0.417 1.000 -ATOM 994 OH2 TIP3 1388 -4.995 -5.474 -0.867 -0.834 1.520 -ATOM 995 H1 TIP3 1388 -4.258 -6.059 -0.762 0.417 1.000 -ATOM 996 H2 TIP3 1388 -5.254 -5.547 -1.812 0.417 1.000 -ATOM 997 OH2 TIP3 1389 -4.573 -6.101 -6.582 -0.834 1.520 -ATOM 998 H1 TIP3 1389 -4.402 -5.297 -6.085 0.417 1.000 -ATOM 999 H2 TIP3 1389 -5.431 -5.968 -7.006 0.417 1.000 -ATOM 1000 OH2 TIP3 1392 -4.883 -3.658 -4.680 -0.834 1.520 -ATOM 1001 H1 TIP3 1392 -5.810 -3.342 -4.765 0.417 1.000 -ATOM 1002 H2 TIP3 1392 -4.275 -2.920 -5.050 0.417 1.000 -ATOM 1003 OH2 TIP3 1394 -6.035 -12.670 8.278 -0.834 1.520 -ATOM 1004 H1 TIP3 1394 -6.373 -11.798 8.654 0.417 1.000 -ATOM 1005 H2 TIP3 1394 -5.866 -13.292 9.029 0.417 1.000 -ATOM 1006 OH2 TIP3 1396 -11.536 0.926 10.643 -0.834 1.520 -ATOM 1007 H1 TIP3 1396 -11.281 1.313 11.498 0.417 1.000 -ATOM 1008 H2 TIP3 1396 -11.523 -0.038 10.810 0.417 1.000 -ATOM 1009 OH2 TIP3 1398 -4.372 -16.112 11.949 -0.834 1.520 -ATOM 1010 H1 TIP3 1398 -4.305 -16.597 11.167 0.417 1.000 -ATOM 1011 H2 TIP3 1398 -4.313 -16.773 12.622 0.417 1.000 -ATOM 1012 OH2 TIP3 1407 -6.737 3.513 -12.674 -0.834 1.520 -ATOM 1013 H1 TIP3 1407 -5.917 3.900 -12.311 0.417 1.000 -ATOM 1014 H2 TIP3 1407 -6.331 2.803 -13.261 0.417 1.000 -ATOM 1015 OH2 TIP3 1408 -15.711 5.675 -18.525 -0.834 1.520 -ATOM 1016 H1 TIP3 1408 -14.986 5.297 -18.091 0.417 1.000 -ATOM 1017 H2 TIP3 1408 -16.372 5.027 -18.487 0.417 1.000 -ATOM 1018 OH2 TIP3 1411 -9.970 -8.559 -16.566 -0.834 1.520 -ATOM 1019 H1 TIP3 1411 -10.726 -9.173 -16.578 0.417 1.000 -ATOM 1020 H2 TIP3 1411 -9.467 -8.638 -17.339 0.417 1.000 -ATOM 1021 OH2 TIP3 1412 -9.450 -10.456 -8.685 -0.834 1.520 -ATOM 1022 H1 TIP3 1412 -9.945 -9.657 -8.604 0.417 1.000 -ATOM 1023 H2 TIP3 1412 -8.540 -10.154 -8.663 0.417 1.000 -ATOM 1024 OH2 TIP3 1413 -2.804 -1.962 -6.223 -0.834 1.520 -ATOM 1025 H1 TIP3 1413 -2.508 -2.915 -6.250 0.417 1.000 -ATOM 1026 H2 TIP3 1413 -3.132 -1.918 -7.145 0.417 1.000 -ATOM 1027 OH2 TIP3 1414 -17.995 -2.204 1.122 -0.834 1.520 -ATOM 1028 H1 TIP3 1414 -17.227 -2.675 1.598 0.417 1.000 -ATOM 1029 H2 TIP3 1414 -17.827 -1.242 1.265 0.417 1.000 -ATOM 1030 OH2 TIP3 1415 -6.447 -4.760 1.435 -0.834 1.520 -ATOM 1031 H1 TIP3 1415 -5.897 -5.041 0.726 0.417 1.000 -ATOM 1032 H2 TIP3 1415 -6.578 -5.530 1.979 0.417 1.000 -ATOM 1033 OH2 TIP3 1416 -12.004 -3.094 8.597 -0.834 1.520 -ATOM 1034 H1 TIP3 1416 -12.612 -3.309 7.856 0.417 1.000 -ATOM 1035 H2 TIP3 1416 -11.729 -2.162 8.310 0.417 1.000 -ATOM 1036 OH2 TIP3 1417 -1.413 -4.224 6.957 -0.834 1.520 -ATOM 1037 H1 TIP3 1417 -2.159 -3.739 6.479 0.417 1.000 -ATOM 1038 H2 TIP3 1417 -1.428 -5.174 6.675 0.417 1.000 -ATOM 1039 OH2 TIP3 1418 -8.544 -17.650 -0.415 -0.834 1.520 -ATOM 1040 H1 TIP3 1418 -9.269 -17.069 -0.270 0.417 1.000 -ATOM 1041 H2 TIP3 1418 -8.454 -18.138 0.401 0.417 1.000 -ATOM 1042 OH2 TIP3 1420 1.969 2.337 11.122 -0.834 1.520 -ATOM 1043 H1 TIP3 1420 1.598 2.329 10.260 0.417 1.000 -ATOM 1044 H2 TIP3 1420 2.906 2.787 11.039 0.417 1.000 -ATOM 1045 OH2 TIP3 1421 -4.360 2.903 18.615 -0.834 1.520 -ATOM 1046 H1 TIP3 1421 -3.757 3.661 18.665 0.417 1.000 -ATOM 1047 H2 TIP3 1421 -3.746 2.148 18.657 0.417 1.000 -ATOM 1048 OH2 TIP3 1428 -6.932 -17.086 -9.851 -0.834 1.520 -ATOM 1049 H1 TIP3 1428 -7.197 -17.527 -10.705 0.417 1.000 -ATOM 1050 H2 TIP3 1428 -5.990 -17.151 -9.888 0.417 1.000 -ATOM 1051 OH2 TIP3 1429 -14.550 -10.933 -14.663 -0.834 1.520 -ATOM 1052 H1 TIP3 1429 -15.278 -11.160 -15.289 0.417 1.000 -ATOM 1053 H2 TIP3 1429 -15.061 -11.230 -13.883 0.417 1.000 -ATOM 1054 OH2 TIP3 1430 -10.187 5.619 -5.270 -0.834 1.520 -ATOM 1055 H1 TIP3 1430 -9.344 5.876 -4.836 0.417 1.000 -ATOM 1056 H2 TIP3 1430 -10.804 6.145 -4.725 0.417 1.000 -ATOM 1057 OH2 TIP3 1431 8.957 1.722 -14.497 -0.834 1.520 -ATOM 1058 H1 TIP3 1431 8.135 1.403 -14.937 0.417 1.000 -ATOM 1059 H2 TIP3 1431 8.661 2.493 -13.925 0.417 1.000 -ATOM 1060 OH2 TIP3 1433 -16.703 -1.229 -2.697 -0.834 1.520 -ATOM 1061 H1 TIP3 1433 -16.087 -0.728 -2.175 0.417 1.000 -ATOM 1062 H2 TIP3 1433 -16.384 -1.065 -3.592 0.417 1.000 -ATOM 1063 OH2 TIP3 1435 -13.403 -4.046 1.463 -0.834 1.520 -ATOM 1064 H1 TIP3 1435 -12.843 -4.840 1.200 0.417 1.000 -ATOM 1065 H2 TIP3 1435 -12.795 -3.549 2.021 0.417 1.000 -ATOM 1066 OH2 TIP3 1436 -15.616 6.376 16.209 -0.834 1.520 -ATOM 1067 H1 TIP3 1436 -15.490 5.829 17.007 0.417 1.000 -ATOM 1068 H2 TIP3 1436 -16.256 7.049 16.456 0.417 1.000 -ATOM 1069 OH2 TIP3 1437 -5.082 -14.990 7.460 -0.834 1.520 -ATOM 1070 H1 TIP3 1437 -5.408 -14.066 7.510 0.417 1.000 -ATOM 1071 H2 TIP3 1437 -4.171 -14.925 7.140 0.417 1.000 -ATOM 1072 OH2 TIP3 1439 -13.438 9.156 9.103 -0.834 1.520 -ATOM 1073 H1 TIP3 1439 -12.443 9.127 9.247 0.417 1.000 -ATOM 1074 H2 TIP3 1439 -13.493 9.741 8.296 0.417 1.000 -ATOM 1075 OH2 TIP3 1440 -18.557 -16.020 -1.373 -0.834 1.520 -ATOM 1076 H1 TIP3 1440 -18.464 -16.950 -1.753 0.417 1.000 -ATOM 1077 H2 TIP3 1440 -19.411 -15.718 -1.744 0.417 1.000 -ATOM 1078 OH2 TIP3 1442 -13.752 -6.495 4.232 -0.834 1.520 -ATOM 1079 H1 TIP3 1442 -12.943 -6.178 3.871 0.417 1.000 -ATOM 1080 H2 TIP3 1442 -13.749 -6.199 5.132 0.417 1.000 -ATOM 1081 OH2 TIP3 1450 0.255 5.983 -16.871 -0.834 1.520 -ATOM 1082 H1 TIP3 1450 0.472 6.411 -16.046 0.417 1.000 -ATOM 1083 H2 TIP3 1450 -0.132 5.098 -16.674 0.417 1.000 -ATOM 1084 OH2 TIP3 1451 -16.752 2.876 -16.299 -0.834 1.520 -ATOM 1085 H1 TIP3 1451 -17.338 2.334 -15.752 0.417 1.000 -ATOM 1086 H2 TIP3 1451 -15.894 2.410 -16.219 0.417 1.000 -ATOM 1087 OH2 TIP3 1453 -2.095 8.143 -13.552 -0.834 1.520 -ATOM 1088 H1 TIP3 1453 -2.826 8.506 -14.038 0.417 1.000 -ATOM 1089 H2 TIP3 1453 -2.526 7.472 -13.020 0.417 1.000 -ATOM 1090 OH2 TIP3 1454 -11.057 -0.393 -10.607 -0.834 1.520 -ATOM 1091 H1 TIP3 1454 -10.110 -0.334 -10.826 0.417 1.000 -ATOM 1092 H2 TIP3 1454 -11.449 -0.798 -11.414 0.417 1.000 -ATOM 1093 OH2 TIP3 1455 -12.466 3.313 2.010 -0.834 1.520 -ATOM 1094 H1 TIP3 1455 -12.691 2.777 2.738 0.417 1.000 -ATOM 1095 H2 TIP3 1455 -13.303 3.403 1.532 0.417 1.000 -ATOM 1096 OH2 TIP3 1457 -6.660 -6.440 -3.523 -0.834 1.520 -ATOM 1097 H1 TIP3 1457 -6.600 -6.432 -4.458 0.417 1.000 -ATOM 1098 H2 TIP3 1457 -7.148 -5.594 -3.367 0.417 1.000 -ATOM 1099 OH2 TIP3 1458 -11.656 -12.155 0.429 -0.834 1.520 -ATOM 1100 H1 TIP3 1458 -11.521 -11.708 1.236 0.417 1.000 -ATOM 1101 H2 TIP3 1458 -10.898 -11.941 -0.100 0.417 1.000 -ATOM 1102 OH2 TIP3 1459 -14.562 11.554 2.819 -0.834 1.520 -ATOM 1103 H1 TIP3 1459 -15.377 11.788 3.335 0.417 1.000 -ATOM 1104 H2 TIP3 1459 -14.989 10.914 2.179 0.417 1.000 -ATOM 1105 OH2 TIP3 1461 -7.688 -11.207 18.101 -0.834 1.520 -ATOM 1106 H1 TIP3 1461 -8.485 -10.954 17.539 0.417 1.000 -ATOM 1107 H2 TIP3 1461 -7.086 -11.483 17.387 0.417 1.000 -ATOM 1108 OH2 TIP3 1466 -16.118 13.117 -18.731 -0.834 1.520 -ATOM 1109 H1 TIP3 1466 -15.703 13.920 -18.369 0.417 1.000 -ATOM 1110 H2 TIP3 1466 -15.370 12.476 -18.741 0.417 1.000 -ATOM 1111 OH2 TIP3 1471 1.892 -10.510 -17.046 -0.834 1.520 -ATOM 1112 H1 TIP3 1471 2.662 -11.035 -16.993 0.417 1.000 -ATOM 1113 H2 TIP3 1471 1.279 -11.090 -17.442 0.417 1.000 -ATOM 1114 OH2 TIP3 1474 -13.099 1.557 -10.674 -0.834 1.520 -ATOM 1115 H1 TIP3 1474 -12.590 0.744 -10.616 0.417 1.000 -ATOM 1116 H2 TIP3 1474 -12.526 2.077 -11.241 0.417 1.000 -ATOM 1117 OH2 TIP3 1475 -3.739 1.789 2.152 -0.834 1.520 -ATOM 1118 H1 TIP3 1475 -3.316 0.873 1.878 0.417 1.000 -ATOM 1119 H2 TIP3 1475 -3.212 2.010 2.873 0.417 1.000 -ATOM 1120 OH2 TIP3 1476 -15.942 -1.794 8.885 -0.834 1.520 -ATOM 1121 H1 TIP3 1476 -16.725 -2.400 8.880 0.417 1.000 -ATOM 1122 H2 TIP3 1476 -15.164 -2.347 9.254 0.417 1.000 -ATOM 1123 OH2 TIP3 1477 -15.471 14.567 1.893 -0.834 1.520 -ATOM 1124 H1 TIP3 1477 -15.650 13.803 1.339 0.417 1.000 -ATOM 1125 H2 TIP3 1477 -15.992 15.319 1.446 0.417 1.000 -ATOM 1126 OH2 TIP3 1478 -3.922 8.219 -3.537 -0.834 1.520 -ATOM 1127 H1 TIP3 1478 -4.730 8.606 -3.810 0.417 1.000 -ATOM 1128 H2 TIP3 1478 -3.513 8.107 -4.419 0.417 1.000 -ATOM 1129 OH2 TIP3 1479 -9.939 5.226 16.361 -0.834 1.520 -ATOM 1130 H1 TIP3 1479 -9.809 6.085 15.854 0.417 1.000 -ATOM 1131 H2 TIP3 1479 -8.950 4.997 16.510 0.417 1.000 -ATOM 1132 OH2 TIP3 1480 -18.159 8.556 -0.519 -0.834 1.520 -ATOM 1133 H1 TIP3 1480 -17.414 8.152 -0.929 0.417 1.000 -ATOM 1134 H2 TIP3 1480 -18.762 8.723 -1.259 0.417 1.000 -ATOM 1135 OH2 TIP3 1481 0.228 4.555 13.339 -0.834 1.520 -ATOM 1136 H1 TIP3 1481 -0.232 3.920 12.719 0.417 1.000 -ATOM 1137 H2 TIP3 1481 -0.548 4.798 13.967 0.417 1.000 -ATOM 1138 OH2 TIP3 1493 -14.404 3.880 -1.238 -0.834 1.520 -ATOM 1139 H1 TIP3 1493 -14.273 4.762 -0.815 0.417 1.000 -ATOM 1140 H2 TIP3 1493 -13.496 3.604 -1.569 0.417 1.000 -ATOM 1141 OH2 TIP3 1494 -5.597 -12.185 -4.788 -0.834 1.520 -ATOM 1142 H1 TIP3 1494 -6.468 -11.794 -5.089 0.417 1.000 -ATOM 1143 H2 TIP3 1494 -5.115 -12.319 -5.627 0.417 1.000 -ATOM 1144 OH2 TIP3 1497 -13.143 -8.862 -13.171 -0.834 1.520 -ATOM 1145 H1 TIP3 1497 -13.893 -8.250 -13.200 0.417 1.000 -ATOM 1146 H2 TIP3 1497 -13.520 -9.613 -13.647 0.417 1.000 -ATOM 1147 OH2 TIP3 1499 -8.243 6.769 0.348 -0.834 1.520 -ATOM 1148 H1 TIP3 1499 -8.536 7.496 0.874 0.417 1.000 -ATOM 1149 H2 TIP3 1499 -9.014 6.221 0.287 0.417 1.000 -ATOM 1150 OH2 TIP3 1500 -3.797 13.061 16.285 -0.834 1.520 -ATOM 1151 H1 TIP3 1500 -4.660 12.808 16.599 0.417 1.000 -ATOM 1152 H2 TIP3 1500 -3.356 13.638 16.927 0.417 1.000 -ATOM 1153 OH2 TIP3 1501 -14.593 -6.037 14.866 -0.834 1.520 -ATOM 1154 H1 TIP3 1501 -14.345 -5.437 15.653 0.417 1.000 -ATOM 1155 H2 TIP3 1501 -14.797 -6.854 15.256 0.417 1.000 -ATOM 1156 OH2 TIP3 1504 -2.847 5.314 10.166 -0.834 1.520 -ATOM 1157 H1 TIP3 1504 -3.540 5.077 9.558 0.417 1.000 -ATOM 1158 H2 TIP3 1504 -3.248 5.889 10.826 0.417 1.000 -ATOM 1159 OH2 TIP3 1506 -2.533 13.017 18.975 -0.834 1.520 -ATOM 1160 H1 TIP3 1506 -2.452 12.147 18.669 0.417 1.000 -ATOM 1161 H2 TIP3 1506 -1.613 13.298 19.206 0.417 1.000 -ATOM 1162 OH2 TIP3 1512 -9.691 1.500 -8.519 -0.834 1.520 -ATOM 1163 H1 TIP3 1512 -8.986 0.928 -8.815 0.417 1.000 -ATOM 1164 H2 TIP3 1512 -10.405 0.961 -8.817 0.417 1.000 -ATOM 1165 OH2 TIP3 1514 -14.767 13.186 -7.181 -0.834 1.520 -ATOM 1166 H1 TIP3 1514 -13.894 13.436 -7.294 0.417 1.000 -ATOM 1167 H2 TIP3 1514 -15.217 13.261 -8.046 0.417 1.000 -ATOM 1168 OH2 TIP3 1515 -3.311 6.229 -11.711 -0.834 1.520 -ATOM 1169 H1 TIP3 1515 -2.808 5.791 -11.000 0.417 1.000 -ATOM 1170 H2 TIP3 1515 -3.998 6.674 -11.148 0.417 1.000 -ATOM 1171 OH2 TIP3 1518 -15.671 7.006 6.531 -0.834 1.520 -ATOM 1172 H1 TIP3 1518 -15.907 7.925 6.417 0.417 1.000 -ATOM 1173 H2 TIP3 1518 -15.237 7.043 7.393 0.417 1.000 -ATOM 1174 OH2 TIP3 1519 -7.629 9.653 -8.110 -0.834 1.520 -ATOM 1175 H1 TIP3 1519 -6.690 9.891 -8.015 0.417 1.000 -ATOM 1176 H2 TIP3 1519 -8.022 10.381 -7.593 0.417 1.000 -ATOM 1177 OH2 TIP3 1520 -9.511 9.412 16.667 -0.834 1.520 -ATOM 1178 H1 TIP3 1520 -9.677 10.348 16.799 0.417 1.000 -ATOM 1179 H2 TIP3 1520 -10.082 9.212 15.870 0.417 1.000 -ATOM 1180 OH2 TIP3 1522 -14.728 11.045 13.344 -0.834 1.520 -ATOM 1181 H1 TIP3 1522 -13.963 11.028 12.684 0.417 1.000 -ATOM 1182 H2 TIP3 1522 -15.335 10.381 12.941 0.417 1.000 -ATOM 1183 OH2 TIP3 1523 -2.958 9.632 9.680 -0.834 1.520 -ATOM 1184 H1 TIP3 1523 -3.300 10.443 10.183 0.417 1.000 -ATOM 1185 H2 TIP3 1523 -3.187 8.905 10.358 0.417 1.000 -ATOM 1186 OH2 TIP3 1531 -6.622 16.017 -9.909 -0.834 1.520 -ATOM 1187 H1 TIP3 1531 -6.156 16.623 -9.399 0.417 1.000 -ATOM 1188 H2 TIP3 1531 -7.042 16.567 -10.595 0.417 1.000 -ATOM 1189 OH2 TIP3 1532 -6.765 2.943 -4.367 -0.834 1.520 -ATOM 1190 H1 TIP3 1532 -7.387 2.525 -3.806 0.417 1.000 -ATOM 1191 H2 TIP3 1532 -5.973 2.981 -3.820 0.417 1.000 -ATOM 1192 OH2 TIP3 1534 -0.764 12.819 -10.109 -0.834 1.520 -ATOM 1193 H1 TIP3 1534 -0.893 12.085 -10.646 0.417 1.000 -ATOM 1194 H2 TIP3 1534 -0.913 12.479 -9.243 0.417 1.000 -ATOM 1195 OH2 TIP3 1537 -4.414 15.909 -7.076 -0.834 1.520 -ATOM 1196 H1 TIP3 1537 -4.308 16.788 -7.440 0.417 1.000 -ATOM 1197 H2 TIP3 1537 -4.326 15.267 -7.850 0.417 1.000 -ATOM 1198 OH2 TIP3 1538 -4.999 12.510 -6.474 -0.834 1.520 -ATOM 1199 H1 TIP3 1538 -4.923 13.137 -7.235 0.417 1.000 -ATOM 1200 H2 TIP3 1538 -4.144 12.607 -6.037 0.417 1.000 -ATOM 1201 OH2 TIP3 1540 -2.268 14.385 -2.122 -0.834 1.520 -ATOM 1202 H1 TIP3 1540 -2.152 15.369 -1.849 0.417 1.000 -ATOM 1203 H2 TIP3 1540 -2.436 13.930 -1.255 0.417 1.000 -ATOM 1204 OH2 TIP3 1542 -6.426 4.360 4.620 -0.834 1.520 -ATOM 1205 H1 TIP3 1542 -5.835 4.812 3.959 0.417 1.000 -ATOM 1206 H2 TIP3 1542 -6.198 3.408 4.433 0.417 1.000 -ATOM 1207 OH2 TIP3 1543 6.491 19.767 15.452 -0.834 1.520 -ATOM 1208 H1 TIP3 1543 7.199 19.779 16.123 0.417 1.000 -ATOM 1209 H2 TIP3 1543 5.683 19.538 15.992 0.417 1.000 -ATOM 1210 OH2 TIP3 1544 2.184 -12.187 18.081 -0.834 1.520 -ATOM 1211 H1 TIP3 1544 1.493 -12.247 17.334 0.417 1.000 -ATOM 1212 H2 TIP3 1544 2.880 -11.697 17.673 0.417 1.000 -ATOM 1213 OH2 TIP3 1545 -3.285 12.366 13.648 -0.834 1.520 -ATOM 1214 H1 TIP3 1545 -3.649 12.423 14.592 0.417 1.000 -ATOM 1215 H2 TIP3 1545 -3.620 13.270 13.287 0.417 1.000 -ATOM 1216 OH2 TIP3 1546 -18.333 14.715 9.782 -0.834 1.520 -ATOM 1217 H1 TIP3 1546 -18.821 13.940 9.481 0.417 1.000 -ATOM 1218 H2 TIP3 1546 -17.750 14.347 10.478 0.417 1.000 -ATOM 1219 OH2 TIP3 1552 -14.000 12.305 -12.945 -0.834 1.520 -ATOM 1220 H1 TIP3 1552 -14.577 12.996 -13.267 0.417 1.000 -ATOM 1221 H2 TIP3 1552 -13.194 12.807 -12.633 0.417 1.000 -ATOM 1222 OH2 TIP3 1557 -17.966 19.107 4.801 -0.834 1.520 -ATOM 1223 H1 TIP3 1557 -18.711 19.408 4.252 0.417 1.000 -ATOM 1224 H2 TIP3 1557 -17.691 19.946 5.280 0.417 1.000 -ATOM 1225 OH2 TIP3 1559 -13.182 0.066 0.150 -0.834 1.520 -ATOM 1226 H1 TIP3 1559 -12.314 0.067 0.626 0.417 1.000 -ATOM 1227 H2 TIP3 1559 -13.031 -0.131 -0.788 0.417 1.000 -ATOM 1228 OH2 TIP3 1560 2.934 18.275 1.316 -0.834 1.520 -ATOM 1229 H1 TIP3 1560 2.474 17.391 1.397 0.417 1.000 -ATOM 1230 H2 TIP3 1560 2.500 18.687 0.537 0.417 1.000 -ATOM 1231 OH2 TIP3 1582 -2.949 15.779 -4.414 -0.834 1.520 -ATOM 1232 H1 TIP3 1582 -3.828 15.409 -4.301 0.417 1.000 -ATOM 1233 H2 TIP3 1582 -2.402 15.311 -3.746 0.417 1.000 -ATOM 1234 OH2 TIP3 1583 7.440 14.981 -8.254 -0.834 1.520 -ATOM 1235 H1 TIP3 1583 6.738 15.624 -8.575 0.417 1.000 -ATOM 1236 H2 TIP3 1583 7.067 14.537 -7.494 0.417 1.000 -ATOM 1237 OH2 TIP3 1587 -7.464 9.439 9.422 -0.834 1.520 -ATOM 1238 H1 TIP3 1587 -7.753 8.800 8.719 0.417 1.000 -ATOM 1239 H2 TIP3 1587 -6.639 9.655 9.068 0.417 1.000 -ATOM 1240 OH2 TIP3 1589 -18.402 17.055 18.588 -0.834 1.520 -ATOM 1241 H1 TIP3 1589 -18.022 17.799 18.061 0.417 1.000 -ATOM 1242 H2 TIP3 1589 -17.666 16.867 19.246 0.417 1.000 -ATOM 1243 OH2 TIP3 1592 -18.836 2.897 10.118 -0.834 1.520 -ATOM 1244 H1 TIP3 1592 -18.008 2.391 9.989 0.417 1.000 -ATOM 1245 H2 TIP3 1592 -19.494 2.586 9.458 0.417 1.000 -ATOM 1246 OH2 TIP3 1601 -4.685 11.721 0.639 -0.834 1.520 -ATOM 1247 H1 TIP3 1601 -5.285 11.546 -0.089 0.417 1.000 -ATOM 1248 H2 TIP3 1601 -3.987 12.225 0.266 0.417 1.000 -ATOM 1249 OH2 TIP3 1603 -6.204 12.788 8.549 -0.834 1.520 -ATOM 1250 H1 TIP3 1603 -6.912 13.020 7.894 0.417 1.000 -ATOM 1251 H2 TIP3 1603 -5.789 11.989 8.184 0.417 1.000 -ATOM 1252 OH2 TIP3 1611 -12.025 0.030 2.745 -0.834 1.520 -ATOM 1253 H1 TIP3 1611 -11.368 0.654 2.435 0.417 1.000 -ATOM 1254 H2 TIP3 1611 -12.559 0.490 3.501 0.417 1.000 -ATOM 1255 OH2 TIP3 1612 -16.334 15.780 12.993 -0.834 1.520 -ATOM 1256 H1 TIP3 1612 -15.393 15.948 13.319 0.417 1.000 -ATOM 1257 H2 TIP3 1612 -16.525 16.536 12.413 0.417 1.000 -ATOM 1258 OH2 TIP3 1618 -16.156 11.380 -5.086 -0.834 1.520 -ATOM 1259 H1 TIP3 1618 -15.487 11.715 -5.699 0.417 1.000 -ATOM 1260 H2 TIP3 1618 -16.018 10.438 -5.108 0.417 1.000 -ATOM 1261 OH2 TIP3 1624 -16.708 19.264 -3.123 -0.834 1.520 -ATOM 1262 H1 TIP3 1624 -16.277 18.402 -3.057 0.417 1.000 -ATOM 1263 H2 TIP3 1624 -16.075 19.831 -2.724 0.417 1.000 -ATOM 1264 OH2 TIP3 1626 -12.675 17.867 16.364 -0.834 1.520 -ATOM 1265 H1 TIP3 1626 -13.504 18.257 16.693 0.417 1.000 -ATOM 1266 H2 TIP3 1626 -12.115 17.917 17.176 0.417 1.000 -ATOM 1267 OH2 TIP3 1629 -19.451 17.374 14.273 -0.834 1.520 -ATOM 1268 H1 TIP3 1629 -19.619 17.772 13.420 0.417 1.000 -ATOM 1269 H2 TIP3 1629 -19.365 16.466 14.117 0.417 1.000 -ATOM 1270 OH2 TIP3 1638 8.131 13.693 -14.588 -0.834 1.520 -ATOM 1271 H1 TIP3 1638 8.039 12.821 -14.245 0.417 1.000 -ATOM 1272 H2 TIP3 1638 7.481 13.666 -15.332 0.417 1.000 -ATOM 1273 OH2 TIP3 1642 -17.998 12.689 6.857 -0.834 1.520 -ATOM 1274 H1 TIP3 1642 -17.350 13.256 7.437 0.417 1.000 -ATOM 1275 H2 TIP3 1642 -18.216 13.292 6.148 0.417 1.000 -ATOM 1276 OH2 TIP3 1643 -14.426 13.132 -3.286 -0.834 1.520 -ATOM 1277 H1 TIP3 1643 -14.876 12.329 -2.987 0.417 1.000 -ATOM 1278 H2 TIP3 1643 -14.921 13.786 -2.794 0.417 1.000 -ATOM 1279 OH2 TIP3 1670 -9.083 -13.361 8.367 -0.834 1.520 -ATOM 1280 H1 TIP3 1670 -8.359 -13.525 7.708 0.417 1.000 -ATOM 1281 H2 TIP3 1670 -8.861 -13.984 9.056 0.417 1.000 -ATOM 1282 OH2 TIP3 1692 -7.307 -18.670 15.462 -0.834 1.520 -ATOM 1283 H1 TIP3 1692 -6.646 -18.123 15.015 0.417 1.000 -ATOM 1284 H2 TIP3 1692 -6.865 -18.925 16.317 0.417 1.000 -ATOM 1285 OH2 TIP3 1707 -7.887 -19.079 -15.058 -0.834 1.520 -ATOM 1286 H1 TIP3 1707 -7.181 -19.365 -14.438 0.417 1.000 -ATOM 1287 H2 TIP3 1707 -8.186 -18.104 -14.777 0.417 1.000 -ATOM 1288 OH2 TIP3 1718 -14.485 -15.471 12.470 -0.834 1.520 -ATOM 1289 H1 TIP3 1718 -15.397 -15.136 12.559 0.417 1.000 -ATOM 1290 H2 TIP3 1718 -14.518 -15.745 11.597 0.417 1.000 -ATOM 1291 OH2 TIP3 1723 -10.803 -15.625 -4.036 -0.834 1.520 -ATOM 1292 H1 TIP3 1723 -10.837 -16.410 -4.613 0.417 1.000 -ATOM 1293 H2 TIP3 1723 -11.484 -15.833 -3.396 0.417 1.000 -ATOM 1294 OH2 TIP3 1725 -17.579 -16.225 -14.942 -0.834 1.520 -ATOM 1295 H1 TIP3 1725 -17.316 -17.066 -15.348 0.417 1.000 -ATOM 1296 H2 TIP3 1725 -16.781 -15.660 -14.952 0.417 1.000 -ATOM 1297 OH2 TIP3 1727 -1.626 -13.429 4.304 -0.834 1.520 -ATOM 1298 H1 TIP3 1727 -2.370 -13.175 3.710 0.417 1.000 -ATOM 1299 H2 TIP3 1727 -2.027 -14.169 4.860 0.417 1.000 -ATOM 1300 OH2 TIP3 1729 -13.206 -13.125 2.997 -0.834 1.520 -ATOM 1301 H1 TIP3 1729 -12.834 -12.185 3.093 0.417 1.000 -ATOM 1302 H2 TIP3 1729 -14.166 -12.971 3.036 0.417 1.000 -ATOM 1303 OH2 TIP3 1730 -8.482 -17.859 -5.165 -0.834 1.520 -ATOM 1304 H1 TIP3 1730 -8.644 -17.231 -5.899 0.417 1.000 -ATOM 1305 H2 TIP3 1730 -7.655 -18.273 -5.417 0.417 1.000 -ATOM 1306 OH2 TIP3 1733 -16.826 -18.416 -3.024 -0.834 1.520 -ATOM 1307 H1 TIP3 1733 -17.630 -17.923 -3.190 0.417 1.000 -ATOM 1308 H2 TIP3 1733 -16.730 -19.039 -3.748 0.417 1.000 -ATOM 1309 OH2 TIP3 1735 3.583 -10.324 16.567 -0.834 1.520 -ATOM 1310 H1 TIP3 1735 3.123 -9.802 15.925 0.417 1.000 -ATOM 1311 H2 TIP3 1735 4.026 -10.951 16.040 0.417 1.000 -ATOM 1312 OH2 TIP3 1744 -0.181 -18.815 -13.579 -0.834 1.520 -ATOM 1313 H1 TIP3 1744 0.354 -18.357 -14.248 0.417 1.000 -ATOM 1314 H2 TIP3 1744 -0.233 -19.768 -13.901 0.417 1.000 -ATOM 1315 OH2 TIP3 1746 -9.152 -9.929 -1.369 -0.834 1.520 -ATOM 1316 H1 TIP3 1746 -9.655 -9.523 -2.093 0.417 1.000 -ATOM 1317 H2 TIP3 1746 -8.337 -9.449 -1.362 0.417 1.000 -ATOM 1318 OH2 TIP3 1750 -12.939 4.972 -17.457 -0.834 1.520 -ATOM 1319 H1 TIP3 1750 -12.015 5.260 -17.563 0.417 1.000 -ATOM 1320 H2 TIP3 1750 -12.849 4.003 -17.555 0.417 1.000 -ATOM 1321 OH2 TIP3 1752 -7.931 -11.234 3.082 -0.834 1.520 -ATOM 1322 H1 TIP3 1752 -7.977 -10.710 3.888 0.417 1.000 -ATOM 1323 H2 TIP3 1752 -8.410 -12.044 3.400 0.417 1.000 -ATOM 1324 OH2 TIP3 1755 1.084 -16.732 14.355 -0.834 1.520 -ATOM 1325 H1 TIP3 1755 0.432 -17.234 13.844 0.417 1.000 -ATOM 1326 H2 TIP3 1755 0.865 -16.909 15.225 0.417 1.000 -ATOM 1327 OH2 TIP3 1757 -11.286 -9.146 6.340 -0.834 1.520 -ATOM 1328 H1 TIP3 1757 -10.309 -9.440 6.364 0.417 1.000 -ATOM 1329 H2 TIP3 1757 -11.601 -9.189 7.287 0.417 1.000 -ATOM 1330 OH2 TIP3 1764 -14.859 -18.389 -19.501 -0.834 1.520 -ATOM 1331 H1 TIP3 1764 -15.303 -18.890 -18.766 0.417 1.000 -ATOM 1332 H2 TIP3 1764 -13.969 -18.698 -19.414 0.417 1.000 -ATOM 1333 OH2 TIP3 1766 -2.069 -5.628 -8.406 -0.834 1.520 -ATOM 1334 H1 TIP3 1766 -2.990 -5.576 -8.713 0.417 1.000 -ATOM 1335 H2 TIP3 1766 -2.074 -5.044 -7.601 0.417 1.000 -ATOM 1336 OH2 TIP3 1771 -17.491 -16.495 -7.090 -0.834 1.520 -ATOM 1337 H1 TIP3 1771 -17.886 -15.862 -7.685 0.417 1.000 -ATOM 1338 H2 TIP3 1771 -17.285 -17.288 -7.617 0.417 1.000 -ATOM 1339 OH2 TIP3 1772 -6.100 -18.734 11.625 -0.834 1.520 -ATOM 1340 H1 TIP3 1772 -5.368 -19.331 11.842 0.417 1.000 -ATOM 1341 H2 TIP3 1772 -5.733 -18.258 10.831 0.417 1.000 -ATOM 1342 OH2 TIP3 1773 -5.646 -18.648 -5.996 -0.834 1.520 -ATOM 1343 H1 TIP3 1773 -5.306 -19.377 -5.435 0.417 1.000 -ATOM 1344 H2 TIP3 1773 -5.427 -17.805 -5.574 0.417 1.000 -ATOM 1345 OH2 TIP3 1775 -18.087 -18.594 7.668 -0.834 1.520 -ATOM 1346 H1 TIP3 1775 -18.279 -18.573 8.637 0.417 1.000 -ATOM 1347 H2 TIP3 1775 -17.661 -17.701 7.574 0.417 1.000 -ATOM 1348 OH2 TIP3 1776 -12.992 -15.476 16.761 -0.834 1.520 -ATOM 1349 H1 TIP3 1776 -13.730 -14.940 17.088 0.417 1.000 -ATOM 1350 H2 TIP3 1776 -12.255 -14.989 17.135 0.417 1.000 -ATOM 1351 OH2 TIP3 1778 -17.562 -13.996 16.100 -0.834 1.520 -ATOM 1352 H1 TIP3 1778 -18.031 -14.475 16.753 0.417 1.000 -ATOM 1353 H2 TIP3 1778 -18.268 -13.982 15.391 0.417 1.000 -ATOM 1354 OH2 TIP3 1788 -5.422 3.489 -2.150 -0.834 1.520 -ATOM 1355 H1 TIP3 1788 -4.524 3.267 -2.139 0.417 1.000 -ATOM 1356 H2 TIP3 1788 -5.641 3.484 -1.229 0.417 1.000 -ATOM 1357 OH2 TIP3 1789 -14.539 -13.246 -18.057 -0.834 1.520 -ATOM 1358 H1 TIP3 1789 -13.588 -13.373 -17.982 0.417 1.000 -ATOM 1359 H2 TIP3 1789 -14.791 -14.044 -18.566 0.417 1.000 -ATOM 1360 OH2 TIP3 1790 -1.937 -5.563 -3.088 -0.834 1.520 -ATOM 1361 H1 TIP3 1790 -1.261 -6.112 -2.627 0.417 1.000 -ATOM 1362 H2 TIP3 1790 -2.816 -6.023 -3.042 0.417 1.000 -ATOM 1363 OH2 TIP3 1794 -4.701 -18.629 4.731 -0.834 1.520 -ATOM 1364 H1 TIP3 1794 -5.144 -18.285 3.949 0.417 1.000 -ATOM 1365 H2 TIP3 1794 -3.772 -18.473 4.527 0.417 1.000 -ATOM 1366 OH2 TIP3 1795 -5.899 -18.098 2.316 -0.834 1.520 -ATOM 1367 H1 TIP3 1795 -5.114 -17.602 2.002 0.417 1.000 -ATOM 1368 H2 TIP3 1795 -5.892 -18.959 1.813 0.417 1.000 -ATOM 1369 OH2 TIP3 1796 1.008 -14.939 -1.543 -0.834 1.520 -ATOM 1370 H1 TIP3 1796 0.384 -14.979 -0.793 0.417 1.000 -ATOM 1371 H2 TIP3 1796 1.508 -15.790 -1.373 0.417 1.000 -ATOM 1372 OH2 TIP3 1798 -5.880 -1.003 13.901 -0.834 1.520 -ATOM 1373 H1 TIP3 1798 -6.725 -0.812 14.327 0.417 1.000 -ATOM 1374 H2 TIP3 1798 -5.312 -0.203 14.158 0.417 1.000 -ATOM 1375 OH2 TIP3 1808 4.934 -9.881 3.588 -0.834 1.520 -ATOM 1376 H1 TIP3 1808 5.124 -10.621 4.125 0.417 1.000 -ATOM 1377 H2 TIP3 1808 4.257 -9.461 4.069 0.417 1.000 -ATOM 1378 OH2 TIP3 1810 10.667 1.477 -18.568 -0.834 1.520 -ATOM 1379 H1 TIP3 1810 10.600 2.083 -17.781 0.417 1.000 -ATOM 1380 H2 TIP3 1810 9.728 1.243 -18.726 0.417 1.000 -ATOM 1381 OH2 TIP3 1811 10.667 -14.998 -12.111 -0.834 1.520 -ATOM 1382 H1 TIP3 1811 10.007 -14.348 -12.442 0.417 1.000 -ATOM 1383 H2 TIP3 1811 11.404 -14.870 -12.754 0.417 1.000 -ATOM 1384 OH2 TIP3 1812 -14.557 -10.544 8.801 -0.834 1.520 -ATOM 1385 H1 TIP3 1812 -15.371 -10.057 8.641 0.417 1.000 -ATOM 1386 H2 TIP3 1812 -14.873 -11.287 9.296 0.417 1.000 -ATOM 1387 OH2 TIP3 1813 -13.858 -8.142 -3.948 -0.834 1.520 -ATOM 1388 H1 TIP3 1813 -14.238 -7.358 -4.325 0.417 1.000 -ATOM 1389 H2 TIP3 1813 -14.434 -8.854 -4.339 0.417 1.000 -ATOM 1390 OH2 TIP3 1814 -8.813 -0.896 -1.319 -0.834 1.520 -ATOM 1391 H1 TIP3 1814 -7.971 -1.434 -1.137 0.417 1.000 -ATOM 1392 H2 TIP3 1814 -9.426 -1.227 -0.636 0.417 1.000 -ATOM 1393 OH2 TIP3 1815 -12.069 -0.265 8.005 -0.834 1.520 -ATOM 1394 H1 TIP3 1815 -12.770 -0.043 7.391 0.417 1.000 -ATOM 1395 H2 TIP3 1815 -12.157 0.396 8.753 0.417 1.000 -ATOM 1396 OH2 TIP3 1816 -10.095 4.448 1.184 -0.834 1.520 -ATOM 1397 H1 TIP3 1816 -9.546 4.369 2.014 0.417 1.000 -ATOM 1398 H2 TIP3 1816 -10.904 3.987 1.368 0.417 1.000 -ATOM 1399 OH2 TIP3 1817 -0.849 -8.233 16.611 -0.834 1.520 -ATOM 1400 H1 TIP3 1817 -1.631 -8.413 17.199 0.417 1.000 -ATOM 1401 H2 TIP3 1817 -1.284 -8.226 15.731 0.417 1.000 -ATOM 1402 OH2 TIP3 1820 -11.199 -16.666 9.609 -0.834 1.520 -ATOM 1403 H1 TIP3 1820 -11.218 -17.615 9.788 0.417 1.000 -ATOM 1404 H2 TIP3 1820 -10.487 -16.583 8.993 0.417 1.000 -ATOM 1405 OH2 TIP3 1821 -4.060 -8.964 16.024 -0.834 1.520 -ATOM 1406 H1 TIP3 1821 -3.587 -9.337 15.247 0.417 1.000 -ATOM 1407 H2 TIP3 1821 -3.651 -9.497 16.672 0.417 1.000 -ATOM 1408 OH2 TIP3 1824 2.523 -17.842 -18.502 -0.834 1.520 -ATOM 1409 H1 TIP3 1824 2.432 -17.928 -19.454 0.417 1.000 -ATOM 1410 H2 TIP3 1824 2.118 -18.679 -18.139 0.417 1.000 -ATOM 1411 OH2 TIP3 1829 -8.220 -6.491 -19.017 -0.834 1.520 -ATOM 1412 H1 TIP3 1829 -7.689 -6.578 -18.206 0.417 1.000 -ATOM 1413 H2 TIP3 1829 -7.621 -6.184 -19.700 0.417 1.000 -ATOM 1414 OH2 TIP3 1830 -6.590 -6.204 -16.794 -0.834 1.520 -ATOM 1415 H1 TIP3 1830 -7.140 -6.658 -16.048 0.417 1.000 -ATOM 1416 H2 TIP3 1830 -5.622 -6.376 -16.514 0.417 1.000 -ATOM 1417 OH2 TIP3 1831 -12.686 -19.772 -10.848 -0.834 1.520 -ATOM 1418 H1 TIP3 1831 -12.553 -19.804 -11.843 0.417 1.000 -ATOM 1419 H2 TIP3 1831 -11.781 -19.664 -10.574 0.417 1.000 -ATOM 1420 OH2 TIP3 1832 -1.569 -10.142 1.140 -0.834 1.520 -ATOM 1421 H1 TIP3 1832 -1.780 -10.080 0.160 0.417 1.000 -ATOM 1422 H2 TIP3 1832 -0.997 -10.962 1.186 0.417 1.000 -ATOM 1423 OH2 TIP3 1833 -10.061 -9.807 17.050 -0.834 1.520 -ATOM 1424 H1 TIP3 1833 -9.895 -9.246 17.880 0.417 1.000 -ATOM 1425 H2 TIP3 1833 -9.832 -9.166 16.364 0.417 1.000 -ATOM 1426 OH2 TIP3 1835 -8.394 7.709 -2.393 -0.834 1.520 -ATOM 1427 H1 TIP3 1835 -8.304 6.784 -2.445 0.417 1.000 -ATOM 1428 H2 TIP3 1835 -8.154 7.850 -1.482 0.417 1.000 -ATOM 1429 OH2 TIP3 1836 -4.069 -9.860 2.479 -0.834 1.520 -ATOM 1430 H1 TIP3 1836 -3.196 -10.049 2.187 0.417 1.000 -ATOM 1431 H2 TIP3 1836 -4.547 -9.573 1.645 0.417 1.000 -ATOM 1432 OH2 TIP3 1837 -10.737 8.274 -7.577 -0.834 1.520 -ATOM 1433 H1 TIP3 1837 -11.054 8.886 -6.952 0.417 1.000 -ATOM 1434 H2 TIP3 1837 -11.203 8.532 -8.415 0.417 1.000 -ATOM 1435 OH2 TIP3 1838 1.198 -8.648 2.162 -0.834 1.520 -ATOM 1436 H1 TIP3 1838 1.433 -9.350 2.754 0.417 1.000 -ATOM 1437 H2 TIP3 1838 0.481 -8.941 1.590 0.417 1.000 -ATOM 1438 OH2 TIP3 1839 -5.597 -13.162 13.628 -0.834 1.520 -ATOM 1439 H1 TIP3 1839 -6.386 -13.482 13.273 0.417 1.000 -ATOM 1440 H2 TIP3 1839 -5.113 -13.994 13.778 0.417 1.000 -ATOM 1441 OH2 TIP3 1840 5.717 -2.953 11.584 -0.834 1.520 -ATOM 1442 H1 TIP3 1840 5.864 -2.065 11.383 0.417 1.000 -ATOM 1443 H2 TIP3 1840 6.567 -3.411 11.607 0.417 1.000 -ATOM 1444 OH2 TIP3 1841 -6.526 -5.814 18.781 -0.834 1.520 -ATOM 1445 H1 TIP3 1841 -7.234 -5.207 19.131 0.417 1.000 -ATOM 1446 H2 TIP3 1841 -6.012 -5.149 18.339 0.417 1.000 -ATOM 1447 OH2 TIP3 1847 -6.620 9.030 -17.190 -0.834 1.520 -ATOM 1448 H1 TIP3 1847 -7.330 9.133 -16.514 0.417 1.000 -ATOM 1449 H2 TIP3 1847 -6.806 9.795 -17.817 0.417 1.000 -ATOM 1450 OH2 TIP3 1850 -9.299 4.822 -12.461 -0.834 1.520 -ATOM 1451 H1 TIP3 1850 -8.375 4.508 -12.384 0.417 1.000 -ATOM 1452 H2 TIP3 1850 -9.623 4.718 -11.524 0.417 1.000 -ATOM 1453 OH2 TIP3 1851 0.846 0.362 -15.259 -0.834 1.520 -ATOM 1454 H1 TIP3 1851 0.810 0.846 -14.356 0.417 1.000 -ATOM 1455 H2 TIP3 1851 0.840 -0.611 -15.065 0.417 1.000 -ATOM 1456 OH2 TIP3 1852 -9.972 4.028 -9.993 -0.834 1.520 -ATOM 1457 H1 TIP3 1852 -9.925 3.132 -9.576 0.417 1.000 -ATOM 1458 H2 TIP3 1852 -9.960 4.627 -9.215 0.417 1.000 -ATOM 1459 OH2 TIP3 1853 -19.711 -9.462 -6.532 -0.834 1.520 -ATOM 1460 H1 TIP3 1853 -19.714 -9.063 -7.398 0.417 1.000 -ATOM 1461 H2 TIP3 1853 -18.738 -9.643 -6.331 0.417 1.000 -ATOM 1462 OH2 TIP3 1854 -3.786 -12.153 -15.234 -0.834 1.520 -ATOM 1463 H1 TIP3 1854 -4.746 -12.008 -14.839 0.417 1.000 -ATOM 1464 H2 TIP3 1854 -4.095 -12.608 -16.004 0.417 1.000 -ATOM 1465 OH2 TIP3 1856 -7.560 -7.593 -14.630 -0.834 1.520 -ATOM 1466 H1 TIP3 1856 -8.348 -8.113 -14.895 0.417 1.000 -ATOM 1467 H2 TIP3 1856 -7.107 -8.133 -14.023 0.417 1.000 -ATOM 1468 OH2 TIP3 1858 -16.664 -12.238 14.144 -0.834 1.520 -ATOM 1469 H1 TIP3 1858 -17.444 -12.364 13.586 0.417 1.000 -ATOM 1470 H2 TIP3 1858 -16.795 -12.882 14.833 0.417 1.000 -ATOM 1471 OH2 TIP3 1859 -15.649 -12.220 10.737 -0.834 1.520 -ATOM 1472 H1 TIP3 1859 -16.581 -12.269 11.041 0.417 1.000 -ATOM 1473 H2 TIP3 1859 -15.250 -12.906 11.271 0.417 1.000 -ATOM 1474 OH2 TIP3 1860 -8.664 -2.573 5.469 -0.834 1.520 -ATOM 1475 H1 TIP3 1860 -7.881 -2.952 5.850 0.417 1.000 -ATOM 1476 H2 TIP3 1860 -8.757 -3.114 4.615 0.417 1.000 -ATOM 1477 OH2 TIP3 1861 -1.303 -3.425 3.068 -0.834 1.520 -ATOM 1478 H1 TIP3 1861 -1.992 -3.221 3.727 0.417 1.000 -ATOM 1479 H2 TIP3 1861 -1.772 -4.017 2.454 0.417 1.000 -ATOM 1480 OH2 TIP3 1862 15.117 -15.618 13.692 -0.834 1.520 -ATOM 1481 H1 TIP3 1862 14.931 -16.509 13.482 0.417 1.000 -ATOM 1482 H2 TIP3 1862 15.883 -15.701 14.254 0.417 1.000 -ATOM 1483 OH2 TIP3 1866 -7.163 -4.566 9.756 -0.834 1.520 -ATOM 1484 H1 TIP3 1866 -7.336 -3.576 9.827 0.417 1.000 -ATOM 1485 H2 TIP3 1866 -8.058 -4.939 9.951 0.417 1.000 -ATOM 1486 OH2 TIP3 1870 -1.188 9.340 -17.251 -0.834 1.520 -ATOM 1487 H1 TIP3 1870 -1.117 9.394 -16.323 0.417 1.000 -ATOM 1488 H2 TIP3 1870 -2.069 8.859 -17.453 0.417 1.000 -ATOM 1489 OH2 TIP3 1871 -13.032 -8.371 -17.647 -0.834 1.520 -ATOM 1490 H1 TIP3 1871 -13.425 -9.104 -18.193 0.417 1.000 -ATOM 1491 H2 TIP3 1871 -12.917 -7.710 -18.410 0.417 1.000 -ATOM 1492 OH2 TIP3 1872 10.797 -4.682 -13.217 -0.834 1.520 -ATOM 1493 H1 TIP3 1872 11.156 -3.952 -12.761 0.417 1.000 -ATOM 1494 H2 TIP3 1872 10.134 -4.353 -13.790 0.417 1.000 -ATOM 1495 OH2 TIP3 1873 -6.467 -9.450 -12.840 -0.834 1.520 -ATOM 1496 H1 TIP3 1873 -7.392 -9.691 -12.510 0.417 1.000 -ATOM 1497 H2 TIP3 1873 -5.992 -9.060 -12.075 0.417 1.000 -ATOM 1498 OH2 TIP3 1874 -6.957 -1.169 2.933 -0.834 1.520 -ATOM 1499 H1 TIP3 1874 -6.561 -2.064 2.986 0.417 1.000 -ATOM 1500 H2 TIP3 1874 -7.861 -1.236 3.145 0.417 1.000 -ATOM 1501 OH2 TIP3 1875 -7.132 4.879 -15.674 -0.834 1.520 -ATOM 1502 H1 TIP3 1875 -6.822 5.390 -14.883 0.417 1.000 -ATOM 1503 H2 TIP3 1875 -6.811 4.009 -15.497 0.417 1.000 -ATOM 1504 OH2 TIP3 1876 -9.869 8.853 1.944 -0.834 1.520 -ATOM 1505 H1 TIP3 1876 -10.567 9.536 2.151 0.417 1.000 -ATOM 1506 H2 TIP3 1876 -9.650 8.482 2.801 0.417 1.000 -ATOM 1507 OH2 TIP3 1877 2.780 12.600 -3.433 -0.834 1.520 -ATOM 1508 H1 TIP3 1877 2.151 12.517 -2.703 0.417 1.000 -ATOM 1509 H2 TIP3 1877 3.032 11.657 -3.633 0.417 1.000 -ATOM 1510 OH2 TIP3 1878 3.306 -12.189 2.076 -0.834 1.520 -ATOM 1511 H1 TIP3 1878 2.635 -12.114 2.789 0.417 1.000 -ATOM 1512 H2 TIP3 1878 3.960 -12.834 2.410 0.417 1.000 -ATOM 1513 OH2 TIP3 1880 -19.664 -6.521 13.304 -0.834 1.520 -ATOM 1514 H1 TIP3 1880 -19.936 -6.277 12.406 0.417 1.000 -ATOM 1515 H2 TIP3 1880 -18.688 -6.483 13.284 0.417 1.000 -ATOM 1516 OH2 TIP3 1881 -2.292 0.770 -7.131 -0.834 1.520 -ATOM 1517 H1 TIP3 1881 -3.296 0.781 -7.220 0.417 1.000 -ATOM 1518 H2 TIP3 1881 -2.106 -0.052 -6.677 0.417 1.000 -ATOM 1519 OH2 TIP3 1883 -12.137 3.614 15.172 -0.834 1.520 -ATOM 1520 H1 TIP3 1883 -12.664 3.211 15.892 0.417 1.000 -ATOM 1521 H2 TIP3 1883 -11.620 4.370 15.576 0.417 1.000 -ATOM 1522 OH2 TIP3 1885 4.964 -6.124 -17.131 -0.834 1.520 -ATOM 1523 H1 TIP3 1885 4.211 -6.801 -17.127 0.417 1.000 -ATOM 1524 H2 TIP3 1885 5.157 -5.944 -18.090 0.417 1.000 -ATOM 1525 OH2 TIP3 1891 -8.938 9.274 -15.814 -0.834 1.520 -ATOM 1526 H1 TIP3 1891 -9.767 9.104 -15.328 0.417 1.000 -ATOM 1527 H2 TIP3 1891 -8.264 9.011 -15.161 0.417 1.000 -ATOM 1528 OH2 TIP3 1892 -15.121 14.045 -10.003 -0.834 1.520 -ATOM 1529 H1 TIP3 1892 -14.817 14.410 -10.863 0.417 1.000 -ATOM 1530 H2 TIP3 1892 -14.645 14.696 -9.366 0.417 1.000 -ATOM 1531 OH2 TIP3 1894 -8.294 -4.189 -3.171 -0.834 1.520 -ATOM 1532 H1 TIP3 1894 -9.108 -3.653 -3.410 0.417 1.000 -ATOM 1533 H2 TIP3 1894 -7.656 -3.534 -2.838 0.417 1.000 -ATOM 1534 OH2 TIP3 1895 6.537 -6.096 -4.508 -0.834 1.520 -ATOM 1535 H1 TIP3 1895 6.365 -5.498 -3.823 0.417 1.000 -ATOM 1536 H2 TIP3 1895 7.409 -6.518 -4.307 0.417 1.000 -ATOM 1537 OH2 TIP3 1896 3.194 0.211 -9.587 -0.834 1.520 -ATOM 1538 H1 TIP3 1896 3.287 -0.138 -8.660 0.417 1.000 -ATOM 1539 H2 TIP3 1896 3.903 0.869 -9.727 0.417 1.000 -ATOM 1540 OH2 TIP3 1897 -17.001 3.837 0.047 -0.834 1.520 -ATOM 1541 H1 TIP3 1897 -17.415 4.567 -0.381 0.417 1.000 -ATOM 1542 H2 TIP3 1897 -16.033 3.961 -0.137 0.417 1.000 -ATOM 1543 OH2 TIP3 1898 -16.276 5.584 3.543 -0.834 1.520 -ATOM 1544 H1 TIP3 1898 -16.065 6.464 3.222 0.417 1.000 -ATOM 1545 H2 TIP3 1898 -15.690 5.508 4.314 0.417 1.000 -ATOM 1546 OH2 TIP3 1899 -3.739 -2.221 15.664 -0.834 1.520 -ATOM 1547 H1 TIP3 1899 -2.936 -2.009 15.207 0.417 1.000 -ATOM 1548 H2 TIP3 1899 -4.393 -1.736 15.157 0.417 1.000 -ATOM 1549 OH2 TIP3 1900 -5.479 -3.404 4.056 -0.834 1.520 -ATOM 1550 H1 TIP3 1900 -5.626 -4.000 4.808 0.417 1.000 -ATOM 1551 H2 TIP3 1900 -5.945 -3.868 3.347 0.417 1.000 -ATOM 1552 OH2 TIP3 1901 -8.459 -3.672 14.335 -0.834 1.520 -ATOM 1553 H1 TIP3 1901 -8.693 -3.158 15.143 0.417 1.000 -ATOM 1554 H2 TIP3 1901 -7.488 -3.790 14.437 0.417 1.000 -ATOM 1555 OH2 TIP3 1902 -10.571 1.891 13.233 -0.834 1.520 -ATOM 1556 H1 TIP3 1902 -10.952 2.477 13.914 0.417 1.000 -ATOM 1557 H2 TIP3 1902 -10.056 2.530 12.713 0.417 1.000 -ATOM 1558 OH2 TIP3 1907 0.732 -4.433 17.945 -0.834 1.520 -ATOM 1559 H1 TIP3 1907 0.341 -4.361 17.048 0.417 1.000 -ATOM 1560 H2 TIP3 1907 1.017 -5.336 17.919 0.417 1.000 -ATOM 1561 OH2 TIP3 1911 -8.358 -0.970 -15.134 -0.834 1.520 -ATOM 1562 H1 TIP3 1911 -9.248 -1.214 -14.791 0.417 1.000 -ATOM 1563 H2 TIP3 1911 -8.466 -0.853 -16.103 0.417 1.000 -ATOM 1564 OH2 TIP3 1912 -6.450 1.522 -10.435 -0.834 1.520 -ATOM 1565 H1 TIP3 1912 -6.721 1.744 -11.345 0.417 1.000 -ATOM 1566 H2 TIP3 1912 -6.141 2.419 -10.086 0.417 1.000 -ATOM 1567 OH2 TIP3 1913 1.097 12.860 -15.300 -0.834 1.520 -ATOM 1568 H1 TIP3 1913 0.853 13.778 -15.395 0.417 1.000 -ATOM 1569 H2 TIP3 1913 0.276 12.355 -15.391 0.417 1.000 -ATOM 1570 OH2 TIP3 1914 -7.999 0.877 -5.892 -0.834 1.520 -ATOM 1571 H1 TIP3 1914 -7.940 1.566 -5.226 0.417 1.000 -ATOM 1572 H2 TIP3 1914 -8.814 1.104 -6.371 0.417 1.000 -ATOM 1573 OH2 TIP3 1915 -7.186 -2.094 -12.617 -0.834 1.520 -ATOM 1574 H1 TIP3 1915 -6.621 -2.590 -13.312 0.417 1.000 -ATOM 1575 H2 TIP3 1915 -8.037 -2.006 -13.046 0.417 1.000 -ATOM 1576 OH2 TIP3 1916 -10.837 14.477 4.848 -0.834 1.520 -ATOM 1577 H1 TIP3 1916 -10.206 13.719 4.654 0.417 1.000 -ATOM 1578 H2 TIP3 1916 -11.499 14.351 4.107 0.417 1.000 -ATOM 1579 OH2 TIP3 1918 -18.636 14.101 -10.624 -0.834 1.520 -ATOM 1580 H1 TIP3 1918 -19.578 14.237 -10.695 0.417 1.000 -ATOM 1581 H2 TIP3 1918 -18.247 14.697 -11.307 0.417 1.000 -ATOM 1582 OH2 TIP3 1919 -5.991 5.677 1.729 -0.834 1.520 -ATOM 1583 H1 TIP3 1919 -5.731 4.939 1.201 0.417 1.000 -ATOM 1584 H2 TIP3 1919 -6.828 5.892 1.272 0.417 1.000 -ATOM 1585 OH2 TIP3 1920 -9.504 -1.732 16.274 -0.834 1.520 -ATOM 1586 H1 TIP3 1920 -9.534 -1.267 17.138 0.417 1.000 -ATOM 1587 H2 TIP3 1920 -10.283 -2.299 16.279 0.417 1.000 -ATOM 1588 OH2 TIP3 1922 2.183 -6.931 10.851 -0.834 1.520 -ATOM 1589 H1 TIP3 1922 2.615 -6.077 10.919 0.417 1.000 -ATOM 1590 H2 TIP3 1922 2.806 -7.386 10.265 0.417 1.000 -ATOM 1591 OH2 TIP3 1923 -11.890 10.743 2.992 -0.834 1.520 -ATOM 1592 H1 TIP3 1923 -12.832 10.903 2.969 0.417 1.000 -ATOM 1593 H2 TIP3 1923 -11.794 10.251 3.816 0.417 1.000 -ATOM 1594 OH2 TIP3 1924 -2.643 12.575 5.351 -0.834 1.520 -ATOM 1595 H1 TIP3 1924 -3.299 13.201 4.994 0.417 1.000 -ATOM 1596 H2 TIP3 1924 -1.975 13.149 5.772 0.417 1.000 -ATOM 1597 OH2 TIP3 1925 -15.575 1.890 18.126 -0.834 1.520 -ATOM 1598 H1 TIP3 1925 -15.751 1.639 19.106 0.417 1.000 -ATOM 1599 H2 TIP3 1925 -16.079 1.119 17.675 0.417 1.000 -ATOM 1600 OH2 TIP3 1934 -2.567 14.841 -15.855 -0.834 1.520 -ATOM 1601 H1 TIP3 1934 -3.249 15.502 -15.543 0.417 1.000 -ATOM 1602 H2 TIP3 1934 -3.028 14.395 -16.596 0.417 1.000 -ATOM 1603 OH2 TIP3 1935 -2.005 5.202 -9.577 -0.834 1.520 -ATOM 1604 H1 TIP3 1935 -1.147 5.548 -9.739 0.417 1.000 -ATOM 1605 H2 TIP3 1935 -1.842 4.442 -8.926 0.417 1.000 -ATOM 1606 OH2 TIP3 1936 -4.312 5.619 -4.974 -0.834 1.520 -ATOM 1607 H1 TIP3 1936 -5.292 5.775 -5.087 0.417 1.000 -ATOM 1608 H2 TIP3 1936 -4.274 4.688 -5.303 0.417 1.000 -ATOM 1609 OH2 TIP3 1937 -8.556 9.458 -4.532 -0.834 1.520 -ATOM 1610 H1 TIP3 1937 -8.311 9.004 -3.728 0.417 1.000 -ATOM 1611 H2 TIP3 1937 -9.486 9.271 -4.551 0.417 1.000 -ATOM 1612 OH2 TIP3 1938 -3.989 -14.895 1.062 -0.834 1.520 -ATOM 1613 H1 TIP3 1938 -3.911 -15.833 1.099 0.417 1.000 -ATOM 1614 H2 TIP3 1938 -4.866 -14.848 0.694 0.417 1.000 -ATOM 1615 OH2 TIP3 1939 -17.322 11.993 3.775 -0.834 1.520 -ATOM 1616 H1 TIP3 1939 -17.819 11.299 4.246 0.417 1.000 -ATOM 1617 H2 TIP3 1939 -18.064 12.527 3.342 0.417 1.000 -ATOM 1618 OH2 TIP3 1940 -0.923 -3.763 11.800 -0.834 1.520 -ATOM 1619 H1 TIP3 1940 -1.492 -3.933 10.961 0.417 1.000 -ATOM 1620 H2 TIP3 1940 -0.095 -3.448 11.352 0.417 1.000 -ATOM 1621 OH2 TIP3 1941 -3.741 -1.558 8.650 -0.834 1.520 -ATOM 1622 H1 TIP3 1941 -3.953 -1.183 9.527 0.417 1.000 -ATOM 1623 H2 TIP3 1941 -3.478 -0.771 8.190 0.417 1.000 -ATOM 1624 OH2 TIP3 1942 -2.708 16.170 16.891 -0.834 1.520 -ATOM 1625 H1 TIP3 1942 -2.724 16.320 17.868 0.417 1.000 -ATOM 1626 H2 TIP3 1942 -3.571 16.626 16.655 0.417 1.000 -ATOM 1627 OH2 TIP3 1943 -0.442 8.328 18.936 -0.834 1.520 -ATOM 1628 H1 TIP3 1943 -0.864 8.281 18.086 0.417 1.000 -ATOM 1629 H2 TIP3 1943 -1.186 8.573 19.541 0.417 1.000 -ATOM 1630 OH2 TIP3 1946 -11.257 18.619 12.721 -0.834 1.520 -ATOM 1631 H1 TIP3 1946 -11.058 18.923 13.577 0.417 1.000 -ATOM 1632 H2 TIP3 1946 -11.002 17.661 12.717 0.417 1.000 -ATOM 1633 OH2 TIP3 1948 -8.810 -7.634 19.187 -0.834 1.520 -ATOM 1634 H1 TIP3 1948 -8.799 -8.186 19.956 0.417 1.000 -ATOM 1635 H2 TIP3 1948 -7.880 -7.309 19.132 0.417 1.000 -ATOM 1636 OH2 TIP3 1949 -17.129 0.653 9.415 -0.834 1.520 -ATOM 1637 H1 TIP3 1949 -16.885 -0.221 9.135 0.417 1.000 -ATOM 1638 H2 TIP3 1949 -17.883 0.410 9.894 0.417 1.000 -ATOM 1639 OH2 TIP3 1950 -4.268 8.840 -15.046 -0.834 1.520 -ATOM 1640 H1 TIP3 1950 -5.208 9.023 -15.188 0.417 1.000 -ATOM 1641 H2 TIP3 1950 -4.095 8.211 -15.757 0.417 1.000 -ATOM 1642 OH2 TIP3 1952 -4.011 -0.249 -15.138 -0.834 1.520 -ATOM 1643 H1 TIP3 1952 -4.048 0.632 -15.540 0.417 1.000 -ATOM 1644 H2 TIP3 1952 -4.443 -0.790 -15.812 0.417 1.000 -ATOM 1645 OH2 TIP3 1953 -8.866 12.630 -16.006 -0.834 1.520 -ATOM 1646 H1 TIP3 1953 -9.542 12.295 -15.277 0.417 1.000 -ATOM 1647 H2 TIP3 1953 -8.833 13.577 -15.879 0.417 1.000 -ATOM 1648 OH2 TIP3 1954 -9.795 10.663 -11.399 -0.834 1.520 -ATOM 1649 H1 TIP3 1954 -9.771 10.950 -10.452 0.417 1.000 -ATOM 1650 H2 TIP3 1954 -9.056 10.051 -11.491 0.417 1.000 -ATOM 1651 OH2 TIP3 1955 -13.615 5.655 -12.517 -0.834 1.520 -ATOM 1652 H1 TIP3 1955 -13.415 5.839 -13.461 0.417 1.000 -ATOM 1653 H2 TIP3 1955 -14.275 6.337 -12.270 0.417 1.000 -ATOM 1654 OH2 TIP3 1961 -8.983 3.800 3.596 -0.834 1.520 -ATOM 1655 H1 TIP3 1961 -9.236 3.363 4.471 0.417 1.000 -ATOM 1656 H2 TIP3 1961 -8.032 4.032 3.763 0.417 1.000 -ATOM 1657 OH2 TIP3 1962 -3.147 9.921 4.396 -0.834 1.520 -ATOM 1658 H1 TIP3 1962 -3.759 10.299 3.746 0.417 1.000 -ATOM 1659 H2 TIP3 1962 -2.915 10.590 5.060 0.417 1.000 -ATOM 1660 OH2 TIP3 1963 1.960 13.969 13.910 -0.834 1.520 -ATOM 1661 H1 TIP3 1963 2.544 13.341 14.372 0.417 1.000 -ATOM 1662 H2 TIP3 1963 1.055 13.872 14.298 0.417 1.000 -ATOM 1663 OH2 TIP3 1965 -17.882 -7.577 15.784 -0.834 1.520 -ATOM 1664 H1 TIP3 1965 -18.643 -8.169 15.813 0.417 1.000 -ATOM 1665 H2 TIP3 1965 -17.560 -7.729 14.837 0.417 1.000 -ATOM 1666 OH2 TIP3 1967 -14.082 -4.166 9.930 -0.834 1.520 -ATOM 1667 H1 TIP3 1967 -13.393 -3.443 9.684 0.417 1.000 -ATOM 1668 H2 TIP3 1967 -14.172 -4.191 10.917 0.417 1.000 -ATOM 1669 OH2 TIP3 1976 0.293 14.856 -18.461 -0.834 1.520 -ATOM 1670 H1 TIP3 1976 0.002 14.771 -19.393 0.417 1.000 -ATOM 1671 H2 TIP3 1976 -0.071 14.068 -18.034 0.417 1.000 -ATOM 1672 OH2 TIP3 1977 3.688 3.202 -15.897 -0.834 1.520 -ATOM 1673 H1 TIP3 1977 3.403 2.394 -16.449 0.417 1.000 -ATOM 1674 H2 TIP3 1977 3.793 3.867 -16.621 0.417 1.000 -ATOM 1675 OH2 TIP3 1979 0.290 19.044 -4.402 -0.834 1.520 -ATOM 1676 H1 TIP3 1979 -0.666 19.056 -4.347 0.417 1.000 -ATOM 1677 H2 TIP3 1979 0.536 19.607 -5.161 0.417 1.000 -ATOM 1678 OH2 TIP3 1980 -7.219 12.593 11.693 -0.834 1.520 -ATOM 1679 H1 TIP3 1980 -8.070 12.353 11.168 0.417 1.000 -ATOM 1680 H2 TIP3 1980 -6.924 13.438 11.352 0.417 1.000 -ATOM 1681 OH2 TIP3 1982 -1.033 6.471 16.810 -0.834 1.520 -ATOM 1682 H1 TIP3 1982 -0.864 5.649 17.228 0.417 1.000 -ATOM 1683 H2 TIP3 1982 -1.290 6.229 15.951 0.417 1.000 -ATOM 1684 OH2 TIP3 1984 -10.044 11.919 17.094 -0.834 1.520 -ATOM 1685 H1 TIP3 1984 -9.839 12.883 17.257 0.417 1.000 -ATOM 1686 H2 TIP3 1984 -10.116 11.612 18.042 0.417 1.000 -ATOM 1687 OH2 TIP3 1986 -12.087 17.648 9.840 -0.834 1.520 -ATOM 1688 H1 TIP3 1986 -11.545 18.374 9.431 0.417 1.000 -ATOM 1689 H2 TIP3 1986 -12.019 17.883 10.756 0.417 1.000 -ATOM 1690 OH2 TIP3 1987 -5.591 -0.672 5.695 -0.834 1.520 -ATOM 1691 H1 TIP3 1987 -5.747 -0.940 6.626 0.417 1.000 -ATOM 1692 H2 TIP3 1987 -6.207 -1.191 5.183 0.417 1.000 -ATOM 1693 OH2 TIP3 1988 2.230 8.195 19.629 -0.834 1.520 -ATOM 1694 H1 TIP3 1988 2.614 7.339 19.356 0.417 1.000 -ATOM 1695 H2 TIP3 1988 1.337 8.266 19.164 0.417 1.000 -ATOM 1696 OH2 TIP3 1989 -0.265 17.590 15.776 -0.834 1.520 -ATOM 1697 H1 TIP3 1989 -0.976 17.031 15.347 0.417 1.000 -ATOM 1698 H2 TIP3 1989 -0.736 18.245 16.281 0.417 1.000 -ATOM 1699 OH2 TIP3 1993 2.691 9.073 -13.850 -0.834 1.520 -ATOM 1700 H1 TIP3 1993 3.249 8.638 -14.462 0.417 1.000 -ATOM 1701 H2 TIP3 1993 2.694 8.441 -13.081 0.417 1.000 -ATOM 1702 OH2 TIP3 1998 -10.258 15.426 -11.763 -0.834 1.520 -ATOM 1703 H1 TIP3 1998 -9.767 15.615 -10.952 0.417 1.000 -ATOM 1704 H2 TIP3 1998 -9.577 15.170 -12.483 0.417 1.000 -ATOM 1705 OH2 TIP3 1999 -10.521 13.182 1.768 -0.834 1.520 -ATOM 1706 H1 TIP3 1999 -11.307 13.725 1.761 0.417 1.000 -ATOM 1707 H2 TIP3 1999 -10.814 12.339 2.131 0.417 1.000 -ATOM 1708 OH2 TIP3 2002 -4.653 10.170 -7.971 -0.834 1.520 -ATOM 1709 H1 TIP3 2002 -4.265 9.648 -8.701 0.417 1.000 -ATOM 1710 H2 TIP3 2002 -3.918 10.398 -7.418 0.417 1.000 -ATOM 1711 OH2 TIP3 2003 -10.261 11.436 6.578 -0.834 1.520 -ATOM 1712 H1 TIP3 2003 -10.819 10.653 6.654 0.417 1.000 -ATOM 1713 H2 TIP3 2003 -9.862 11.376 5.663 0.417 1.000 -ATOM 1714 OH2 TIP3 2009 0.569 -0.349 18.788 -0.834 1.520 -ATOM 1715 H1 TIP3 2009 0.947 -1.166 19.205 0.417 1.000 -ATOM 1716 H2 TIP3 2009 0.693 -0.485 17.847 0.417 1.000 -ATOM 1717 OH2 TIP3 2014 -3.319 13.631 -18.200 -0.834 1.520 -ATOM 1718 H1 TIP3 2014 -3.374 13.886 -19.115 0.417 1.000 -ATOM 1719 H2 TIP3 2014 -4.118 13.213 -17.996 0.417 1.000 -ATOM 1720 OH2 TIP3 2018 -6.159 19.123 -9.510 -0.834 1.520 -ATOM 1721 H1 TIP3 2018 -5.738 19.328 -10.337 0.417 1.000 -ATOM 1722 H2 TIP3 2018 -6.643 19.915 -9.274 0.417 1.000 -ATOM 1723 OH2 TIP3 2020 0.221 11.597 7.386 -0.834 1.520 -ATOM 1724 H1 TIP3 2020 0.133 11.804 8.277 0.417 1.000 -ATOM 1725 H2 TIP3 2020 0.202 10.634 7.321 0.417 1.000 -ATOM 1726 OH2 TIP3 2021 -6.150 11.054 -4.066 -0.834 1.520 -ATOM 1727 H1 TIP3 2021 -5.824 11.533 -4.844 0.417 1.000 -ATOM 1728 H2 TIP3 2021 -6.692 10.414 -4.550 0.417 1.000 -ATOM 1729 OH2 TIP3 2022 -16.341 16.945 -6.365 -0.834 1.520 -ATOM 1730 H1 TIP3 2022 -16.631 16.694 -7.252 0.417 1.000 -ATOM 1731 H2 TIP3 2022 -15.389 16.868 -6.432 0.417 1.000 -ATOM 1732 OH2 TIP3 2025 -16.274 14.553 4.475 -0.834 1.520 -ATOM 1733 H1 TIP3 2025 -15.892 14.814 3.599 0.417 1.000 -ATOM 1734 H2 TIP3 2025 -16.633 13.696 4.203 0.417 1.000 -ATOM 1735 OH2 TIP3 2026 5.565 12.679 17.388 -0.834 1.520 -ATOM 1736 H1 TIP3 2026 6.267 12.035 17.075 0.417 1.000 -ATOM 1737 H2 TIP3 2026 5.491 13.252 16.591 0.417 1.000 -ATOM 1738 OH2 TIP3 2031 9.438 16.373 18.993 -0.834 1.520 -ATOM 1739 H1 TIP3 2031 9.976 16.804 18.343 0.417 1.000 -ATOM 1740 H2 TIP3 2031 10.075 16.399 19.741 0.417 1.000 -ATOM 1741 OH2 TIP3 2037 -7.470 17.675 -12.217 -0.834 1.520 -ATOM 1742 H1 TIP3 2037 -6.758 18.363 -12.110 0.417 1.000 -ATOM 1743 H2 TIP3 2037 -8.324 18.280 -12.242 0.417 1.000 -ATOM 1744 OH2 TIP3 2039 2.501 18.806 -19.404 -0.834 1.520 -ATOM 1745 H1 TIP3 2039 1.645 18.424 -19.104 0.417 1.000 -ATOM 1746 H2 TIP3 2039 2.458 19.666 -18.894 0.417 1.000 -ATOM 1747 OH2 TIP3 2040 -7.838 17.850 -16.009 -0.834 1.520 -ATOM 1748 H1 TIP3 2040 -7.166 18.534 -15.664 0.417 1.000 -ATOM 1749 H2 TIP3 2040 -8.684 18.141 -15.559 0.417 1.000 -ATOM 1750 OH2 TIP3 2042 1.676 16.469 -4.678 -0.834 1.520 -ATOM 1751 H1 TIP3 2042 1.602 16.118 -3.791 0.417 1.000 -ATOM 1752 H2 TIP3 2042 1.363 17.388 -4.652 0.417 1.000 -ATOM 1753 OH2 TIP3 2043 15.399 18.905 -8.489 -0.834 1.520 -ATOM 1754 H1 TIP3 2043 15.834 18.074 -8.723 0.417 1.000 -ATOM 1755 H2 TIP3 2043 15.700 19.072 -7.575 0.417 1.000 -ATOM 1756 OH2 TIP3 2044 -2.679 19.308 0.360 -0.834 1.520 -ATOM 1757 H1 TIP3 2044 -3.518 19.745 0.448 0.417 1.000 -ATOM 1758 H2 TIP3 2044 -2.015 19.946 0.715 0.417 1.000 -ATOM 1759 OH2 TIP3 2061 -2.461 18.628 -4.607 -0.834 1.520 -ATOM 1760 H1 TIP3 2061 -2.718 18.910 -5.451 0.417 1.000 -ATOM 1761 H2 TIP3 2061 -2.514 17.693 -4.700 0.417 1.000 -ATOM 1762 OH2 TIP3 2070 -11.282 15.755 13.150 -0.834 1.520 -ATOM 1763 H1 TIP3 2070 -11.730 15.309 13.875 0.417 1.000 -ATOM 1764 H2 TIP3 2070 -11.419 15.078 12.442 0.417 1.000 -ATOM 1765 OH2 TIP3 2079 -4.571 17.194 -16.923 -0.834 1.520 -ATOM 1766 H1 TIP3 2079 -5.173 16.701 -17.396 0.417 1.000 -ATOM 1767 H2 TIP3 2079 -4.744 17.007 -15.937 0.417 1.000 -ATOM 1768 OH2 TIP3 2082 2.263 18.788 -16.029 -0.834 1.520 -ATOM 1769 H1 TIP3 2082 3.127 19.127 -16.167 0.417 1.000 -ATOM 1770 H2 TIP3 2082 1.659 19.087 -16.835 0.417 1.000 -ATOM 1771 OH2 TIP3 2083 -18.479 17.571 -4.668 -0.834 1.520 -ATOM 1772 H1 TIP3 2083 -17.758 17.486 -5.349 0.417 1.000 -ATOM 1773 H2 TIP3 2083 -18.060 18.181 -3.995 0.417 1.000 -ATOM 1774 OH2 TIP3 2086 3.402 17.593 4.419 -0.834 1.520 -ATOM 1775 H1 TIP3 2086 3.148 16.633 4.549 0.417 1.000 -ATOM 1776 H2 TIP3 2086 3.908 17.810 5.218 0.417 1.000 -ATOM 1777 OH2 TIP3 2105 -11.927 8.386 -4.115 -0.834 1.520 -ATOM 1778 H1 TIP3 2105 -12.129 8.913 -4.903 0.417 1.000 -ATOM 1779 H2 TIP3 2105 -11.792 9.108 -3.427 0.417 1.000 -ATOM 1780 OH2 TIP3 2123 -9.037 -18.263 -8.210 -0.834 1.520 -ATOM 1781 H1 TIP3 2123 -9.681 -17.760 -8.666 0.417 1.000 -ATOM 1782 H2 TIP3 2123 -8.291 -17.833 -8.790 0.417 1.000 -ATOM 1783 OH2 TIP3 2127 -19.018 -12.588 5.939 -0.834 1.520 -ATOM 1784 H1 TIP3 2127 -18.758 -11.970 5.208 0.417 1.000 -ATOM 1785 H2 TIP3 2127 -19.040 -13.477 5.504 0.417 1.000 -ATOM 1786 OH2 TIP3 2133 -5.207 -15.408 18.528 -0.834 1.520 -ATOM 1787 H1 TIP3 2133 -5.550 -14.893 17.758 0.417 1.000 -ATOM 1788 H2 TIP3 2133 -5.166 -14.771 19.194 0.417 1.000 -ATOM 1789 OH2 TIP3 2147 -11.718 -12.192 7.208 -0.834 1.520 -ATOM 1790 H1 TIP3 2147 -10.787 -11.843 7.568 0.417 1.000 -ATOM 1791 H2 TIP3 2147 -12.108 -12.539 8.013 0.417 1.000 -ATOM 1792 OH2 TIP3 2151 -1.294 -18.993 18.225 -0.834 1.520 -ATOM 1793 H1 TIP3 2151 -2.025 -19.609 18.331 0.417 1.000 -ATOM 1794 H2 TIP3 2151 -1.579 -18.233 18.756 0.417 1.000 -ATOM 1795 OH2 TIP3 2155 8.045 -11.616 11.621 -0.834 1.520 -ATOM 1796 H1 TIP3 2155 7.356 -11.039 11.192 0.417 1.000 -ATOM 1797 H2 TIP3 2155 7.459 -12.073 12.301 0.417 1.000 -ATOM 1798 OH2 TIP3 2163 -14.474 -11.631 -5.142 -0.834 1.520 -ATOM 1799 H1 TIP3 2163 -15.050 -11.782 -5.928 0.417 1.000 -ATOM 1800 H2 TIP3 2163 -15.055 -11.081 -4.637 0.417 1.000 -ATOM 1801 OH2 TIP3 2165 -5.224 -15.061 -7.557 -0.834 1.520 -ATOM 1802 H1 TIP3 2165 -5.364 -14.048 -7.748 0.417 1.000 -ATOM 1803 H2 TIP3 2165 -6.135 -15.288 -7.396 0.417 1.000 -ATOM 1804 OH2 TIP3 2167 -12.471 -15.492 -18.604 -0.834 1.520 -ATOM 1805 H1 TIP3 2167 -11.687 -15.418 -19.143 0.417 1.000 -ATOM 1806 H2 TIP3 2167 -13.217 -15.489 -19.232 0.417 1.000 -ATOM 1807 OH2 TIP3 2172 -8.565 -19.164 2.059 -0.834 1.520 -ATOM 1808 H1 TIP3 2172 -8.142 -19.897 1.657 0.417 1.000 -ATOM 1809 H2 TIP3 2172 -7.781 -18.719 2.508 0.417 1.000 -ATOM 1810 OH2 TIP3 2185 -0.992 -17.803 -0.139 -0.834 1.520 -ATOM 1811 H1 TIP3 2185 -1.064 -16.914 -0.575 0.417 1.000 -ATOM 1812 H2 TIP3 2185 -0.468 -18.289 -0.839 0.417 1.000 -ATOM 1813 OH2 TIP3 2186 4.048 -14.612 -3.267 -0.834 1.520 -ATOM 1814 H1 TIP3 2186 4.171 -14.139 -2.391 0.417 1.000 -ATOM 1815 H2 TIP3 2186 3.125 -14.629 -3.401 0.417 1.000 -ATOM 1816 OH2 TIP3 2187 4.212 -18.286 -7.867 -0.834 1.520 -ATOM 1817 H1 TIP3 2187 3.582 -18.639 -7.201 0.417 1.000 -ATOM 1818 H2 TIP3 2187 4.341 -19.001 -8.475 0.417 1.000 -ATOM 1819 OH2 TIP3 2195 10.295 -12.380 9.842 -0.834 1.520 -ATOM 1820 H1 TIP3 2195 9.590 -12.662 9.214 0.417 1.000 -ATOM 1821 H2 TIP3 2195 9.871 -12.139 10.617 0.417 1.000 -ATOM 1822 OH2 TIP3 2208 12.275 -3.562 -9.308 -0.834 1.520 -ATOM 1823 H1 TIP3 2208 12.945 -4.242 -9.368 0.417 1.000 -ATOM 1824 H2 TIP3 2208 12.204 -3.287 -10.258 0.417 1.000 -ATOM 1825 OH2 TIP3 2211 -2.076 -15.320 6.419 -0.834 1.520 -ATOM 1826 H1 TIP3 2211 -1.138 -15.470 6.316 0.417 1.000 -ATOM 1827 H2 TIP3 2211 -2.169 -15.289 7.395 0.417 1.000 -ATOM 1828 OH2 TIP3 2212 1.339 -2.977 10.377 -0.834 1.520 -ATOM 1829 H1 TIP3 2212 1.748 -3.374 9.690 0.417 1.000 -ATOM 1830 H2 TIP3 2212 1.134 -2.070 10.068 0.417 1.000 -ATOM 1831 OH2 TIP3 2213 14.913 -4.945 5.651 -0.834 1.520 -ATOM 1832 H1 TIP3 2213 14.238 -4.945 6.364 0.417 1.000 -ATOM 1833 H2 TIP3 2213 15.296 -4.025 5.770 0.417 1.000 -ATOM 1834 OH2 TIP3 2217 -3.525 -10.834 13.855 -0.834 1.520 -ATOM 1835 H1 TIP3 2217 -4.559 -10.784 13.831 0.417 1.000 -ATOM 1836 H2 TIP3 2217 -3.275 -10.906 12.897 0.417 1.000 -ATOM 1837 OH2 TIP3 2219 -6.451 -13.680 16.634 -0.834 1.520 -ATOM 1838 H1 TIP3 2219 -5.971 -13.032 16.104 0.417 1.000 -ATOM 1839 H2 TIP3 2219 -6.985 -14.124 15.911 0.417 1.000 -ATOM 1840 OH2 TIP3 2227 -8.800 -15.849 -16.529 -0.834 1.520 -ATOM 1841 H1 TIP3 2227 -7.947 -15.640 -16.072 0.417 1.000 -ATOM 1842 H2 TIP3 2227 -9.428 -15.574 -15.862 0.417 1.000 -ATOM 1843 OH2 TIP3 2229 0.914 -1.712 -10.772 -0.834 1.520 -ATOM 1844 H1 TIP3 2229 1.719 -1.532 -10.181 0.417 1.000 -ATOM 1845 H2 TIP3 2229 1.059 -1.125 -11.484 0.417 1.000 -ATOM 1846 OH2 TIP3 2231 -15.116 -15.919 -6.287 -0.834 1.520 -ATOM 1847 H1 TIP3 2231 -15.030 -15.254 -5.627 0.417 1.000 -ATOM 1848 H2 TIP3 2231 -16.068 -16.025 -6.407 0.417 1.000 -ATOM 1849 OH2 TIP3 2232 -0.433 -6.220 -0.713 -0.834 1.520 -ATOM 1850 H1 TIP3 2232 0.031 -5.318 -0.607 0.417 1.000 -ATOM 1851 H2 TIP3 2232 0.202 -6.764 -1.255 0.417 1.000 -ATOM 1852 OH2 TIP3 2234 1.592 -11.839 6.970 -0.834 1.520 -ATOM 1853 H1 TIP3 2234 1.978 -12.733 7.048 0.417 1.000 -ATOM 1854 H2 TIP3 2234 1.533 -11.492 7.881 0.417 1.000 -ATOM 1855 OH2 TIP3 2235 10.123 -12.130 17.078 -0.834 1.520 -ATOM 1856 H1 TIP3 2235 10.906 -11.757 16.625 0.417 1.000 -ATOM 1857 H2 TIP3 2235 10.369 -12.269 17.997 0.417 1.000 -ATOM 1858 OH2 TIP3 2236 -18.204 -3.353 6.522 -0.834 1.520 -ATOM 1859 H1 TIP3 2236 -17.562 -2.620 6.694 0.417 1.000 -ATOM 1860 H2 TIP3 2236 -18.069 -3.974 7.267 0.417 1.000 -ATOM 1861 OH2 TIP3 2240 -3.666 -17.737 13.909 -0.834 1.520 -ATOM 1862 H1 TIP3 2240 -3.853 -18.542 14.509 0.417 1.000 -ATOM 1863 H2 TIP3 2240 -2.748 -17.919 13.641 0.417 1.000 -ATOM 1864 OH2 TIP3 2244 -6.133 -15.376 -16.046 -0.834 1.520 -ATOM 1865 H1 TIP3 2244 -5.933 -15.359 -15.096 0.417 1.000 -ATOM 1866 H2 TIP3 2244 -5.878 -14.506 -16.352 0.417 1.000 -ATOM 1867 OH2 TIP3 2250 -2.578 -16.600 -15.798 -0.834 1.520 -ATOM 1868 H1 TIP3 2250 -1.805 -16.566 -16.402 0.417 1.000 -ATOM 1869 H2 TIP3 2250 -2.603 -17.575 -15.593 0.417 1.000 -ATOM 1870 OH2 TIP3 2251 -3.309 -2.084 -13.241 -0.834 1.520 -ATOM 1871 H1 TIP3 2251 -3.594 -1.447 -13.908 0.417 1.000 -ATOM 1872 H2 TIP3 2251 -3.185 -1.474 -12.418 0.417 1.000 -ATOM 1873 OH2 TIP3 2253 -1.807 -2.030 -10.250 -0.834 1.520 -ATOM 1874 H1 TIP3 2253 -2.042 -1.089 -10.167 0.417 1.000 -ATOM 1875 H2 TIP3 2253 -0.849 -2.010 -10.353 0.417 1.000 -ATOM 1876 OH2 TIP3 2254 -2.573 -11.540 -4.938 -0.834 1.520 -ATOM 1877 H1 TIP3 2254 -3.541 -11.477 -4.957 0.417 1.000 -ATOM 1878 H2 TIP3 2254 -2.363 -12.231 -5.612 0.417 1.000 -ATOM 1879 OH2 TIP3 2256 -4.655 -9.948 7.391 -0.834 1.520 -ATOM 1880 H1 TIP3 2256 -3.854 -10.164 7.773 0.417 1.000 -ATOM 1881 H2 TIP3 2256 -5.349 -10.127 8.112 0.417 1.000 -ATOM 1882 OH2 TIP3 2257 -10.328 -19.412 4.858 -0.834 1.520 -ATOM 1883 H1 TIP3 2257 -11.052 -19.706 5.377 0.417 1.000 -ATOM 1884 H2 TIP3 2257 -10.659 -18.594 4.518 0.417 1.000 -ATOM 1885 OH2 TIP3 2258 2.420 -13.866 14.132 -0.834 1.520 -ATOM 1886 H1 TIP3 2258 1.893 -14.665 14.236 0.417 1.000 -ATOM 1887 H2 TIP3 2258 3.154 -14.232 13.532 0.417 1.000 -ATOM 1888 OH2 TIP3 2269 0.640 -12.012 0.638 -0.834 1.520 -ATOM 1889 H1 TIP3 2269 0.659 -12.056 -0.305 0.417 1.000 -ATOM 1890 H2 TIP3 2269 1.535 -11.696 0.911 0.417 1.000 -ATOM 1891 OH2 TIP3 2270 -16.826 -8.841 -14.046 -0.834 1.520 -ATOM 1892 H1 TIP3 2270 -16.096 -8.927 -14.648 0.417 1.000 -ATOM 1893 H2 TIP3 2270 -16.796 -7.896 -13.871 0.417 1.000 -ATOM 1894 OH2 TIP3 2271 4.035 -8.563 -9.058 -0.834 1.520 -ATOM 1895 H1 TIP3 2271 4.407 -7.653 -9.161 0.417 1.000 -ATOM 1896 H2 TIP3 2271 3.483 -8.658 -9.848 0.417 1.000 -ATOM 1897 OH2 TIP3 2272 -7.078 -17.089 -2.588 -0.834 1.520 -ATOM 1898 H1 TIP3 2272 -7.579 -17.569 -1.911 0.417 1.000 -ATOM 1899 H2 TIP3 2272 -7.760 -17.004 -3.299 0.417 1.000 -ATOM 1900 OH2 TIP3 2273 9.572 4.909 -1.184 -0.834 1.520 -ATOM 1901 H1 TIP3 2273 10.207 5.395 -0.688 0.417 1.000 -ATOM 1902 H2 TIP3 2273 10.024 4.065 -1.256 0.417 1.000 -ATOM 1903 OH2 TIP3 2274 -6.245 10.376 -1.340 -0.834 1.520 -ATOM 1904 H1 TIP3 2274 -7.199 10.330 -1.190 0.417 1.000 -ATOM 1905 H2 TIP3 2274 -6.116 10.626 -2.222 0.417 1.000 -ATOM 1906 OH2 TIP3 2275 0.843 -4.997 -11.408 -0.834 1.520 -ATOM 1907 H1 TIP3 2275 1.616 -4.345 -11.476 0.417 1.000 -ATOM 1908 H2 TIP3 2275 0.048 -4.481 -11.445 0.417 1.000 -ATOM 1909 OH2 TIP3 2276 2.830 14.458 0.419 -0.834 1.520 -ATOM 1910 H1 TIP3 2276 3.519 14.206 1.114 0.417 1.000 -ATOM 1911 H2 TIP3 2276 2.353 15.136 0.871 0.417 1.000 -ATOM 1912 OH2 TIP3 2277 -6.356 -12.039 0.946 -0.834 1.520 -ATOM 1913 H1 TIP3 2277 -6.817 -12.842 0.680 0.417 1.000 -ATOM 1914 H2 TIP3 2277 -6.982 -11.685 1.687 0.417 1.000 -ATOM 1915 OH2 TIP3 2278 2.665 -14.370 6.951 -0.834 1.520 -ATOM 1916 H1 TIP3 2278 2.656 -15.139 7.490 0.417 1.000 -ATOM 1917 H2 TIP3 2278 3.484 -14.508 6.420 0.417 1.000 -ATOM 1918 OH2 TIP3 2280 -13.388 -16.604 -2.958 -0.834 1.520 -ATOM 1919 H1 TIP3 2280 -13.862 -17.013 -2.170 0.417 1.000 -ATOM 1920 H2 TIP3 2280 -13.752 -17.117 -3.694 0.417 1.000 -ATOM 1921 OH2 TIP3 2281 -10.082 -8.222 10.774 -0.834 1.520 -ATOM 1922 H1 TIP3 2281 -10.248 -7.298 10.633 0.417 1.000 -ATOM 1923 H2 TIP3 2281 -10.696 -8.689 10.145 0.417 1.000 -ATOM 1924 OH2 TIP3 2283 -14.205 -4.682 17.290 -0.834 1.520 -ATOM 1925 H1 TIP3 2283 -15.161 -4.549 17.478 0.417 1.000 -ATOM 1926 H2 TIP3 2283 -13.947 -5.411 17.884 0.417 1.000 -ATOM 1927 OH2 TIP3 2284 -5.706 -3.601 13.982 -0.834 1.520 -ATOM 1928 H1 TIP3 2284 -5.784 -2.691 13.609 0.417 1.000 -ATOM 1929 H2 TIP3 2284 -4.729 -3.777 13.919 0.417 1.000 -ATOM 1930 OH2 TIP3 2286 -11.658 -3.462 17.080 -0.834 1.520 -ATOM 1931 H1 TIP3 2286 -11.133 -4.147 17.501 0.417 1.000 -ATOM 1932 H2 TIP3 2286 -12.579 -3.840 16.954 0.417 1.000 -ATOM 1933 OH2 TIP3 2288 -16.372 -4.547 -10.316 -0.834 1.520 -ATOM 1934 H1 TIP3 2288 -16.396 -3.632 -10.635 0.417 1.000 -ATOM 1935 H2 TIP3 2288 -16.734 -4.522 -9.408 0.417 1.000 -ATOM 1936 OH2 TIP3 2290 -18.388 12.495 -15.466 -0.834 1.520 -ATOM 1937 H1 TIP3 2290 -18.076 11.828 -14.819 0.417 1.000 -ATOM 1938 H2 TIP3 2290 -19.322 12.557 -15.269 0.417 1.000 -ATOM 1939 OH2 TIP3 2292 -0.540 -14.409 -5.555 -0.834 1.520 -ATOM 1940 H1 TIP3 2292 0.196 -13.750 -5.355 0.417 1.000 -ATOM 1941 H2 TIP3 2292 -0.906 -14.125 -6.419 0.417 1.000 -ATOM 1942 OH2 TIP3 2293 -18.359 -1.736 -11.864 -0.834 1.520 -ATOM 1943 H1 TIP3 2293 -19.245 -1.510 -11.992 0.417 1.000 -ATOM 1944 H2 TIP3 2293 -18.019 -0.997 -11.335 0.417 1.000 -ATOM 1945 OH2 TIP3 2294 9.495 3.138 -4.110 -0.834 1.520 -ATOM 1946 H1 TIP3 2294 9.139 2.254 -3.841 0.417 1.000 -ATOM 1947 H2 TIP3 2294 10.438 2.912 -4.208 0.417 1.000 -ATOM 1948 OH2 TIP3 2295 -11.284 -19.181 1.571 -0.834 1.520 -ATOM 1949 H1 TIP3 2295 -11.641 -19.601 2.460 0.417 1.000 -ATOM 1950 H2 TIP3 2295 -10.378 -18.954 1.690 0.417 1.000 -ATOM 1951 OH2 TIP3 2296 7.308 -9.359 -7.457 -0.834 1.520 -ATOM 1952 H1 TIP3 2296 7.329 -9.716 -8.400 0.417 1.000 -ATOM 1953 H2 TIP3 2296 7.336 -8.403 -7.626 0.417 1.000 -ATOM 1954 OH2 TIP3 2297 3.117 -0.907 8.405 -0.834 1.520 -ATOM 1955 H1 TIP3 2297 2.229 -0.819 8.079 0.417 1.000 -ATOM 1956 H2 TIP3 2297 3.353 -0.032 8.715 0.417 1.000 -ATOM 1957 OH2 TIP3 2298 3.212 -14.197 10.846 -0.834 1.520 -ATOM 1958 H1 TIP3 2298 3.803 -14.486 11.557 0.417 1.000 -ATOM 1959 H2 TIP3 2298 3.808 -13.865 10.158 0.417 1.000 -ATOM 1960 OH2 TIP3 2299 -6.566 5.337 13.390 -0.834 1.520 -ATOM 1961 H1 TIP3 2299 -6.418 4.364 13.530 0.417 1.000 -ATOM 1962 H2 TIP3 2299 -7.137 5.449 12.588 0.417 1.000 -ATOM 1963 OH2 TIP3 2300 -5.531 1.877 4.235 -0.834 1.520 -ATOM 1964 H1 TIP3 2300 -4.676 1.872 3.803 0.417 1.000 -ATOM 1965 H2 TIP3 2300 -5.575 0.993 4.640 0.417 1.000 -ATOM 1966 OH2 TIP3 2301 -9.635 -10.289 13.459 -0.834 1.520 -ATOM 1967 H1 TIP3 2301 -9.266 -10.294 12.552 0.417 1.000 -ATOM 1968 H2 TIP3 2301 -9.163 -9.574 13.909 0.417 1.000 -ATOM 1969 OH2 TIP3 2302 -4.891 -6.546 4.094 -0.834 1.520 -ATOM 1970 H1 TIP3 2302 -4.467 -7.374 4.413 0.417 1.000 -ATOM 1971 H2 TIP3 2302 -4.218 -6.101 3.586 0.417 1.000 -ATOM 1972 OH2 TIP3 2309 -5.731 -9.999 -15.631 -0.834 1.520 -ATOM 1973 H1 TIP3 2309 -5.027 -9.472 -15.248 0.417 1.000 -ATOM 1974 H2 TIP3 2309 -6.310 -10.141 -14.908 0.417 1.000 -ATOM 1975 OH2 TIP3 2311 -5.309 -6.183 -10.950 -0.834 1.520 -ATOM 1976 H1 TIP3 2311 -4.838 -5.628 -11.615 0.417 1.000 -ATOM 1977 H2 TIP3 2311 -6.179 -5.750 -10.887 0.417 1.000 -ATOM 1978 OH2 TIP3 2313 8.308 -12.249 -8.799 -0.834 1.520 -ATOM 1979 H1 TIP3 2313 8.564 -13.175 -9.084 0.417 1.000 -ATOM 1980 H2 TIP3 2313 8.125 -12.349 -7.860 0.417 1.000 -ATOM 1981 OH2 TIP3 2315 -0.749 4.031 -4.981 -0.834 1.520 -ATOM 1982 H1 TIP3 2315 -0.584 4.218 -4.033 0.417 1.000 -ATOM 1983 H2 TIP3 2315 0.120 3.940 -5.437 0.417 1.000 -ATOM 1984 OH2 TIP3 2316 -7.727 -14.576 -9.596 -0.834 1.520 -ATOM 1985 H1 TIP3 2316 -7.616 -15.528 -9.790 0.417 1.000 -ATOM 1986 H2 TIP3 2316 -6.847 -14.286 -9.499 0.417 1.000 -ATOM 1987 OH2 TIP3 2317 -16.241 -6.747 -4.857 -0.834 1.520 -ATOM 1988 H1 TIP3 2317 -16.448 -6.454 -3.941 0.417 1.000 -ATOM 1989 H2 TIP3 2317 -17.138 -6.679 -5.260 0.417 1.000 -ATOM 1990 OH2 TIP3 2318 -9.711 4.036 -17.707 -0.834 1.520 -ATOM 1991 H1 TIP3 2318 -9.653 4.942 -17.982 0.417 1.000 -ATOM 1992 H2 TIP3 2318 -8.811 3.864 -17.445 0.417 1.000 -ATOM 1993 OH2 TIP3 2319 -1.292 0.702 -3.058 -0.834 1.520 -ATOM 1994 H1 TIP3 2319 -1.015 1.641 -2.839 0.417 1.000 -ATOM 1995 H2 TIP3 2319 -1.005 0.185 -2.291 0.417 1.000 -ATOM 1996 OH2 TIP3 2320 -1.861 5.548 2.658 -0.834 1.520 -ATOM 1997 H1 TIP3 2320 -2.466 6.233 2.529 0.417 1.000 -ATOM 1998 H2 TIP3 2320 -1.908 5.149 1.779 0.417 1.000 -ATOM 1999 OH2 TIP3 2321 -11.333 5.901 10.752 -0.834 1.520 -ATOM 2000 H1 TIP3 2321 -11.686 5.203 10.087 0.417 1.000 -ATOM 2001 H2 TIP3 2321 -12.053 5.888 11.430 0.417 1.000 -ATOM 2002 OH2 TIP3 2322 5.181 -6.033 8.608 -0.834 1.520 -ATOM 2003 H1 TIP3 2322 5.211 -6.094 9.597 0.417 1.000 -ATOM 2004 H2 TIP3 2322 4.539 -6.789 8.454 0.417 1.000 -ATOM 2005 OH2 TIP3 2326 -6.455 -7.376 16.168 -0.834 1.520 -ATOM 2006 H1 TIP3 2326 -6.485 -6.980 17.055 0.417 1.000 -ATOM 2007 H2 TIP3 2326 -5.799 -8.127 16.247 0.417 1.000 -ATOM 2008 OH2 TIP3 2327 -15.918 4.655 18.495 -0.834 1.520 -ATOM 2009 H1 TIP3 2327 -15.637 3.688 18.391 0.417 1.000 -ATOM 2010 H2 TIP3 2327 -16.860 4.570 18.697 0.417 1.000 -ATOM 2011 OH2 TIP3 2329 5.575 -3.241 -18.058 -0.834 1.520 -ATOM 2012 H1 TIP3 2329 5.444 -3.693 -17.180 0.417 1.000 -ATOM 2013 H2 TIP3 2329 5.764 -4.038 -18.645 0.417 1.000 -ATOM 2014 OH2 TIP3 2331 7.117 -6.596 -12.875 -0.834 1.520 -ATOM 2015 H1 TIP3 2331 7.014 -6.658 -11.921 0.417 1.000 -ATOM 2016 H2 TIP3 2331 7.894 -7.127 -13.063 0.417 1.000 -ATOM 2017 OH2 TIP3 2332 6.050 -1.247 -9.554 -0.834 1.520 -ATOM 2018 H1 TIP3 2332 5.900 -0.327 -9.635 0.417 1.000 -ATOM 2019 H2 TIP3 2332 6.455 -1.484 -10.453 0.417 1.000 -ATOM 2020 OH2 TIP3 2333 -6.898 9.392 -13.686 -0.834 1.520 -ATOM 2021 H1 TIP3 2333 -6.796 10.236 -14.050 0.417 1.000 -ATOM 2022 H2 TIP3 2333 -6.727 9.565 -12.739 0.417 1.000 -ATOM 2023 OH2 TIP3 2335 -12.681 -5.568 -4.985 -0.834 1.520 -ATOM 2024 H1 TIP3 2335 -12.367 -4.681 -5.258 0.417 1.000 -ATOM 2025 H2 TIP3 2335 -12.298 -5.697 -4.067 0.417 1.000 -ATOM 2026 OH2 TIP3 2336 3.587 13.492 -14.424 -0.834 1.520 -ATOM 2027 H1 TIP3 2336 2.950 13.226 -15.085 0.417 1.000 -ATOM 2028 H2 TIP3 2336 3.460 14.481 -14.329 0.417 1.000 -ATOM 2029 OH2 TIP3 2337 -0.839 15.629 -6.246 -0.834 1.520 -ATOM 2030 H1 TIP3 2337 -1.488 15.731 -5.545 0.417 1.000 -ATOM 2031 H2 TIP3 2337 0.026 15.799 -5.787 0.417 1.000 -ATOM 2032 OH2 TIP3 2338 -1.195 12.960 -7.480 -0.834 1.520 -ATOM 2033 H1 TIP3 2338 -0.703 12.167 -7.218 0.417 1.000 -ATOM 2034 H2 TIP3 2338 -0.445 13.610 -7.281 0.417 1.000 -ATOM 2035 OH2 TIP3 2339 -2.997 2.235 -0.803 -0.834 1.520 -ATOM 2036 H1 TIP3 2339 -2.606 1.585 -1.428 0.417 1.000 -ATOM 2037 H2 TIP3 2339 -3.371 1.813 -0.013 0.417 1.000 -ATOM 2038 OH2 TIP3 2340 -8.728 3.285 10.928 -0.834 1.520 -ATOM 2039 H1 TIP3 2340 -8.630 4.190 11.223 0.417 1.000 -ATOM 2040 H2 TIP3 2340 -8.844 3.343 9.964 0.417 1.000 -ATOM 2041 OH2 TIP3 2342 -4.171 -0.461 11.081 -0.834 1.520 -ATOM 2042 H1 TIP3 2342 -4.438 0.513 11.007 0.417 1.000 -ATOM 2043 H2 TIP3 2342 -4.592 -0.785 11.931 0.417 1.000 -ATOM 2044 OH2 TIP3 2345 -6.539 -4.232 6.741 -0.834 1.520 -ATOM 2045 H1 TIP3 2345 -6.391 -3.634 7.510 0.417 1.000 -ATOM 2046 H2 TIP3 2345 -6.600 -5.129 7.165 0.417 1.000 -ATOM 2047 OH2 TIP3 2346 4.597 -9.421 13.207 -0.834 1.520 -ATOM 2048 H1 TIP3 2346 4.222 -8.579 13.469 0.417 1.000 -ATOM 2049 H2 TIP3 2346 5.117 -9.691 13.974 0.417 1.000 -ATOM 2050 OH2 TIP3 2351 -2.710 -0.708 -17.641 -0.834 1.520 -ATOM 2051 H1 TIP3 2351 -2.429 -1.587 -17.928 0.417 1.000 -ATOM 2052 H2 TIP3 2351 -2.752 -0.755 -16.654 0.417 1.000 -ATOM 2053 OH2 TIP3 2352 -1.234 3.787 -16.016 -0.834 1.520 -ATOM 2054 H1 TIP3 2352 -1.015 3.007 -16.543 0.417 1.000 -ATOM 2055 H2 TIP3 2352 -1.564 3.402 -15.218 0.417 1.000 -ATOM 2056 OH2 TIP3 2354 -3.137 11.310 -12.657 -0.834 1.520 -ATOM 2057 H1 TIP3 2354 -2.921 12.272 -12.729 0.417 1.000 -ATOM 2058 H2 TIP3 2354 -2.289 10.861 -12.781 0.417 1.000 -ATOM 2059 OH2 TIP3 2355 -0.934 -3.532 -13.203 -0.834 1.520 -ATOM 2060 H1 TIP3 2355 -1.700 -3.070 -12.811 0.417 1.000 -ATOM 2061 H2 TIP3 2355 -1.256 -3.708 -14.117 0.417 1.000 -ATOM 2062 OH2 TIP3 2357 -16.883 -8.181 -9.930 -0.834 1.520 -ATOM 2063 H1 TIP3 2357 -16.510 -7.504 -10.477 0.417 1.000 -ATOM 2064 H2 TIP3 2357 -16.063 -8.417 -9.461 0.417 1.000 -ATOM 2065 OH2 TIP3 2358 -19.451 -3.607 -9.634 -0.834 1.520 -ATOM 2066 H1 TIP3 2358 -18.708 -3.678 -8.984 0.417 1.000 -ATOM 2067 H2 TIP3 2358 -19.335 -2.717 -10.028 0.417 1.000 -ATOM 2068 OH2 TIP3 2359 1.404 3.372 -6.577 -0.834 1.520 -ATOM 2069 H1 TIP3 2359 2.157 3.832 -6.909 0.417 1.000 -ATOM 2070 H2 TIP3 2359 1.702 2.597 -6.064 0.417 1.000 -ATOM 2071 OH2 TIP3 2360 -9.507 4.964 7.613 -0.834 1.520 -ATOM 2072 H1 TIP3 2360 -8.860 4.305 7.322 0.417 1.000 -ATOM 2073 H2 TIP3 2360 -10.284 4.459 7.898 0.417 1.000 -ATOM 2074 OH2 TIP3 2361 -8.898 7.719 13.804 -0.834 1.520 -ATOM 2075 H1 TIP3 2361 -8.363 8.501 13.492 0.417 1.000 -ATOM 2076 H2 TIP3 2361 -9.724 8.173 14.077 0.417 1.000 -ATOM 2077 OH2 TIP3 2362 -2.377 -15.037 9.019 -0.834 1.520 -ATOM 2078 H1 TIP3 2362 -1.536 -15.413 9.324 0.417 1.000 -ATOM 2079 H2 TIP3 2362 -2.782 -14.477 9.781 0.417 1.000 -ATOM 2080 OH2 TIP3 2363 6.706 -6.590 17.116 -0.834 1.520 -ATOM 2081 H1 TIP3 2363 7.228 -6.121 16.472 0.417 1.000 -ATOM 2082 H2 TIP3 2363 7.053 -7.484 17.045 0.417 1.000 -ATOM 2083 OH2 TIP3 2364 0.166 13.918 15.942 -0.834 1.520 -ATOM 2084 H1 TIP3 2364 1.019 13.915 16.533 0.417 1.000 -ATOM 2085 H2 TIP3 2364 -0.376 14.719 16.250 0.417 1.000 -ATOM 2086 OH2 TIP3 2365 10.249 -3.744 9.525 -0.834 1.520 -ATOM 2087 H1 TIP3 2365 9.874 -2.993 9.132 0.417 1.000 -ATOM 2088 H2 TIP3 2365 9.562 -4.176 10.012 0.417 1.000 -ATOM 2089 OH2 TIP3 2367 -2.385 10.502 16.418 -0.834 1.520 -ATOM 2090 H1 TIP3 2367 -2.516 10.146 15.556 0.417 1.000 -ATOM 2091 H2 TIP3 2367 -2.164 11.456 16.298 0.417 1.000 -ATOM 2092 OH2 TIP3 2375 8.380 1.948 3.685 -0.834 1.520 -ATOM 2093 H1 TIP3 2375 7.883 1.107 3.699 0.417 1.000 -ATOM 2094 H2 TIP3 2375 9.311 1.736 3.879 0.417 1.000 -ATOM 2095 OH2 TIP3 2376 -3.064 6.039 -16.049 -0.834 1.520 -ATOM 2096 H1 TIP3 2376 -3.448 5.515 -15.384 0.417 1.000 -ATOM 2097 H2 TIP3 2376 -2.269 5.524 -16.316 0.417 1.000 -ATOM 2098 OH2 TIP3 2377 -2.653 -13.106 -7.290 -0.834 1.520 -ATOM 2099 H1 TIP3 2377 -2.965 -12.719 -8.069 0.417 1.000 -ATOM 2100 H2 TIP3 2377 -2.312 -13.953 -7.614 0.417 1.000 -ATOM 2101 OH2 TIP3 2379 -0.703 7.146 9.355 -0.834 1.520 -ATOM 2102 H1 TIP3 2379 -0.088 6.961 10.059 0.417 1.000 -ATOM 2103 H2 TIP3 2379 -1.415 6.521 9.489 0.417 1.000 -ATOM 2104 OH2 TIP3 2380 -4.189 -4.711 -12.800 -0.834 1.520 -ATOM 2105 H1 TIP3 2380 -4.386 -5.031 -13.691 0.417 1.000 -ATOM 2106 H2 TIP3 2380 -3.976 -3.802 -13.013 0.417 1.000 -ATOM 2107 OH2 TIP3 2381 -1.066 0.840 13.368 -0.834 1.520 -ATOM 2108 H1 TIP3 2381 -1.229 -0.094 13.498 0.417 1.000 -ATOM 2109 H2 TIP3 2381 -1.088 1.268 14.242 0.417 1.000 -ATOM 2110 OH2 TIP3 2382 5.578 17.279 2.344 -0.834 1.520 -ATOM 2111 H1 TIP3 2382 4.837 17.870 2.407 0.417 1.000 -ATOM 2112 H2 TIP3 2382 5.510 16.800 3.195 0.417 1.000 -ATOM 2113 OH2 TIP3 2383 -9.446 11.776 9.968 -0.834 1.520 -ATOM 2114 H1 TIP3 2383 -9.269 10.864 9.643 0.417 1.000 -ATOM 2115 H2 TIP3 2383 -9.022 12.349 9.323 0.417 1.000 -ATOM 2116 OH2 TIP3 2385 -6.807 14.917 10.198 -0.834 1.520 -ATOM 2117 H1 TIP3 2385 -6.788 14.153 9.673 0.417 1.000 -ATOM 2118 H2 TIP3 2385 -6.598 15.582 9.559 0.417 1.000 -ATOM 2119 OH2 TIP3 2388 5.526 10.079 11.561 -0.834 1.520 -ATOM 2120 H1 TIP3 2388 4.871 10.095 12.202 0.417 1.000 -ATOM 2121 H2 TIP3 2388 6.295 9.737 12.042 0.417 1.000 -ATOM 2122 OH2 TIP3 2397 -12.706 10.336 -15.568 -0.834 1.520 -ATOM 2123 H1 TIP3 2397 -12.458 11.062 -15.086 0.417 1.000 -ATOM 2124 H2 TIP3 2397 -11.880 9.814 -15.585 0.417 1.000 -ATOM 2125 OH2 TIP3 2398 1.177 10.370 -0.880 -0.834 1.520 -ATOM 2126 H1 TIP3 2398 1.205 11.325 -0.802 0.417 1.000 -ATOM 2127 H2 TIP3 2398 0.266 10.095 -0.620 0.417 1.000 -ATOM 2128 OH2 TIP3 2399 -8.446 18.325 -3.695 -0.834 1.520 -ATOM 2129 H1 TIP3 2399 -8.052 17.932 -2.845 0.417 1.000 -ATOM 2130 H2 TIP3 2399 -7.608 18.669 -4.118 0.417 1.000 -ATOM 2131 OH2 TIP3 2400 -5.634 18.228 -4.039 -0.834 1.520 -ATOM 2132 H1 TIP3 2400 -4.768 18.021 -4.330 0.417 1.000 -ATOM 2133 H2 TIP3 2400 -5.924 17.386 -3.652 0.417 1.000 -ATOM 2134 OH2 TIP3 2402 -9.382 2.201 6.019 -0.834 1.520 -ATOM 2135 H1 TIP3 2402 -10.051 2.230 6.749 0.417 1.000 -ATOM 2136 H2 TIP3 2402 -9.057 1.315 6.136 0.417 1.000 -ATOM 2137 OH2 TIP3 2404 -1.613 15.661 14.268 -0.834 1.520 -ATOM 2138 H1 TIP3 2404 -2.181 14.999 14.712 0.417 1.000 -ATOM 2139 H2 TIP3 2404 -1.772 15.487 13.351 0.417 1.000 -ATOM 2140 OH2 TIP3 2405 -15.263 6.898 9.253 -0.834 1.520 -ATOM 2141 H1 TIP3 2405 -16.010 7.512 9.251 0.417 1.000 -ATOM 2142 H2 TIP3 2405 -14.483 7.467 9.109 0.417 1.000 -ATOM 2143 OH2 TIP3 2406 -1.501 -2.004 14.059 -0.834 1.520 -ATOM 2144 H1 TIP3 2406 -0.959 -2.471 14.748 0.417 1.000 -ATOM 2145 H2 TIP3 2406 -1.464 -2.611 13.319 0.417 1.000 -ATOM 2146 OH2 TIP3 2407 8.179 -0.534 16.622 -0.834 1.520 -ATOM 2147 H1 TIP3 2407 7.417 -0.869 16.267 0.417 1.000 -ATOM 2148 H2 TIP3 2407 8.469 0.165 15.942 0.417 1.000 -ATOM 2149 OH2 TIP3 2416 12.086 16.644 -8.961 -0.834 1.520 -ATOM 2150 H1 TIP3 2416 11.700 15.888 -8.421 0.417 1.000 -ATOM 2151 H2 TIP3 2416 11.393 16.767 -9.661 0.417 1.000 -ATOM 2152 OH2 TIP3 2418 4.789 2.892 1.785 -0.834 1.520 -ATOM 2153 H1 TIP3 2418 4.614 1.960 1.659 0.417 1.000 -ATOM 2154 H2 TIP3 2418 4.930 3.181 0.799 0.417 1.000 -ATOM 2155 OH2 TIP3 2419 -0.855 12.320 -17.951 -0.834 1.520 -ATOM 2156 H1 TIP3 2419 -1.012 11.387 -18.018 0.417 1.000 -ATOM 2157 H2 TIP3 2419 -1.748 12.669 -17.667 0.417 1.000 -ATOM 2158 OH2 TIP3 2420 9.550 15.456 2.808 -0.834 1.520 -ATOM 2159 H1 TIP3 2420 8.745 15.698 2.368 0.417 1.000 -ATOM 2160 H2 TIP3 2420 9.944 16.350 3.078 0.417 1.000 -ATOM 2161 OH2 TIP3 2421 -10.363 11.703 -7.639 -0.834 1.520 -ATOM 2162 H1 TIP3 2421 -10.449 12.630 -7.969 0.417 1.000 -ATOM 2163 H2 TIP3 2421 -9.623 11.806 -6.973 0.417 1.000 -ATOM 2164 OH2 TIP3 2422 -4.878 14.283 4.320 -0.834 1.520 -ATOM 2165 H1 TIP3 2422 -5.722 14.109 3.835 0.417 1.000 -ATOM 2166 H2 TIP3 2422 -4.936 15.157 4.611 0.417 1.000 -ATOM 2167 OH2 TIP3 2424 7.445 16.533 16.119 -0.834 1.520 -ATOM 2168 H1 TIP3 2424 7.227 17.325 16.603 0.417 1.000 -ATOM 2169 H2 TIP3 2424 6.724 15.919 16.445 0.417 1.000 -ATOM 2170 OH2 TIP3 2427 11.792 5.068 17.401 -0.834 1.520 -ATOM 2171 H1 TIP3 2427 12.217 5.675 17.997 0.417 1.000 -ATOM 2172 H2 TIP3 2427 12.136 4.216 17.746 0.417 1.000 -ATOM 2173 OH2 TIP3 2436 -3.581 11.379 -15.573 -0.834 1.520 -ATOM 2174 H1 TIP3 2436 -3.497 11.761 -14.716 0.417 1.000 -ATOM 2175 H2 TIP3 2436 -3.663 10.424 -15.426 0.417 1.000 -ATOM 2176 OH2 TIP3 2439 1.444 13.518 3.608 -0.834 1.520 -ATOM 2177 H1 TIP3 2439 2.227 13.781 4.139 0.417 1.000 -ATOM 2178 H2 TIP3 2439 1.497 12.499 3.687 0.417 1.000 -ATOM 2179 OH2 TIP3 2442 1.058 18.464 -1.607 -0.834 1.520 -ATOM 2180 H1 TIP3 2442 0.985 19.446 -1.444 0.417 1.000 -ATOM 2181 H2 TIP3 2442 0.680 18.330 -2.499 0.417 1.000 -ATOM 2182 OH2 TIP3 2443 -5.271 16.895 5.412 -0.834 1.520 -ATOM 2183 H1 TIP3 2443 -5.222 17.735 4.944 0.417 1.000 -ATOM 2184 H2 TIP3 2443 -6.171 16.879 5.716 0.417 1.000 -ATOM 2185 OH2 TIP3 2444 -0.416 8.718 2.611 -0.834 1.520 -ATOM 2186 H1 TIP3 2444 -0.112 8.010 3.173 0.417 1.000 -ATOM 2187 H2 TIP3 2444 -1.398 8.710 2.688 0.417 1.000 -ATOM 2188 OH2 TIP3 2445 4.514 8.408 8.576 -0.834 1.520 -ATOM 2189 H1 TIP3 2445 4.242 9.169 8.106 0.417 1.000 -ATOM 2190 H2 TIP3 2445 4.628 8.696 9.512 0.417 1.000 -ATOM 2191 OH2 TIP3 2449 -3.618 -0.083 17.413 -0.834 1.520 -ATOM 2192 H1 TIP3 2449 -3.813 -0.936 16.955 0.417 1.000 -ATOM 2193 H2 TIP3 2449 -3.551 -0.321 18.329 0.417 1.000 -ATOM 2194 OH2 TIP3 2450 0.487 16.609 18.307 -0.834 1.520 -ATOM 2195 H1 TIP3 2450 0.542 16.359 17.338 0.417 1.000 -ATOM 2196 H2 TIP3 2450 -0.079 17.406 18.228 0.417 1.000 -ATOM 2197 OH2 TIP3 2458 4.953 17.068 -17.085 -0.834 1.520 -ATOM 2198 H1 TIP3 2458 5.615 16.967 -17.798 0.417 1.000 -ATOM 2199 H2 TIP3 2458 4.145 16.516 -17.433 0.417 1.000 -ATOM 2200 OH2 TIP3 2460 3.392 16.149 -14.679 -0.834 1.520 -ATOM 2201 H1 TIP3 2460 2.454 16.253 -14.966 0.417 1.000 -ATOM 2202 H2 TIP3 2460 3.773 16.596 -15.443 0.417 1.000 -ATOM 2203 OH2 TIP3 2463 -10.363 16.331 -0.396 -0.834 1.520 -ATOM 2204 H1 TIP3 2463 -9.532 16.186 -0.017 0.417 1.000 -ATOM 2205 H2 TIP3 2463 -10.348 15.680 -1.118 0.417 1.000 -ATOM 2206 OH2 TIP3 2464 -6.426 11.494 5.789 -0.834 1.520 -ATOM 2207 H1 TIP3 2464 -5.944 11.618 5.000 0.417 1.000 -ATOM 2208 H2 TIP3 2464 -5.751 11.043 6.314 0.417 1.000 -ATOM 2209 OH2 TIP3 2465 5.035 18.136 -0.832 -0.834 1.520 -ATOM 2210 H1 TIP3 2465 4.860 17.789 0.025 0.417 1.000 -ATOM 2211 H2 TIP3 2465 4.744 19.054 -0.665 0.417 1.000 -ATOM 2212 OH2 TIP3 2466 3.435 16.999 18.783 -0.834 1.520 -ATOM 2213 H1 TIP3 2466 2.571 16.794 18.441 0.417 1.000 -ATOM 2214 H2 TIP3 2466 3.944 16.799 18.018 0.417 1.000 -ATOM 2215 OH2 TIP3 2467 -4.399 17.916 13.849 -0.834 1.520 -ATOM 2216 H1 TIP3 2467 -4.353 17.865 12.892 0.417 1.000 -ATOM 2217 H2 TIP3 2467 -4.037 18.810 13.994 0.417 1.000 -ATOM 2218 OH2 TIP3 2469 0.483 8.826 7.557 -0.834 1.520 -ATOM 2219 H1 TIP3 2469 -0.078 8.497 8.339 0.417 1.000 -ATOM 2220 H2 TIP3 2469 -0.208 8.611 6.826 0.417 1.000 -ATOM 2221 OH2 TIP3 2479 -7.319 13.694 3.012 -0.834 1.520 -ATOM 2222 H1 TIP3 2479 -7.486 13.491 2.077 0.417 1.000 -ATOM 2223 H2 TIP3 2479 -7.963 13.127 3.442 0.417 1.000 -ATOM 2224 OH2 TIP3 2482 -12.027 17.457 -19.162 -0.834 1.520 -ATOM 2225 H1 TIP3 2482 -12.045 17.961 -18.300 0.417 1.000 -ATOM 2226 H2 TIP3 2482 -12.865 17.732 -19.609 0.417 1.000 -ATOM 2227 OH2 TIP3 2488 -7.116 19.132 13.204 -0.834 1.520 -ATOM 2228 H1 TIP3 2488 -7.614 18.406 12.723 0.417 1.000 -ATOM 2229 H2 TIP3 2488 -6.371 18.775 13.657 0.417 1.000 -ATOM 2230 OH2 TIP3 2492 -1.234 19.076 17.778 -0.834 1.520 -ATOM 2231 H1 TIP3 2492 -2.182 18.887 17.981 0.417 1.000 -ATOM 2232 H2 TIP3 2492 -1.042 19.797 18.353 0.417 1.000 -ATOM 2233 OH2 TIP3 2501 14.741 18.679 2.991 -0.834 1.520 -ATOM 2234 H1 TIP3 2501 14.837 18.816 2.050 0.417 1.000 -ATOM 2235 H2 TIP3 2501 13.848 18.992 3.178 0.417 1.000 -ATOM 2236 OH2 TIP3 2507 9.142 13.772 11.652 -0.834 1.520 -ATOM 2237 H1 TIP3 2507 8.872 13.113 12.275 0.417 1.000 -ATOM 2238 H2 TIP3 2507 8.448 13.766 10.998 0.417 1.000 -ATOM 2239 OH2 TIP3 2508 -8.796 10.133 -0.556 -0.834 1.520 -ATOM 2240 H1 TIP3 2508 -8.778 11.077 -0.267 0.417 1.000 -ATOM 2241 H2 TIP3 2508 -8.773 9.668 0.308 0.417 1.000 -ATOM 2242 OH2 TIP3 2541 -11.704 14.793 -14.192 -0.834 1.520 -ATOM 2243 H1 TIP3 2541 -12.372 14.702 -14.886 0.417 1.000 -ATOM 2244 H2 TIP3 2541 -11.088 15.487 -14.513 0.417 1.000 -ATOM 2245 OH2 TIP3 2582 -6.695 16.968 -19.228 -0.834 1.520 -ATOM 2246 H1 TIP3 2582 -7.651 17.077 -19.405 0.417 1.000 -ATOM 2247 H2 TIP3 2582 -6.267 17.791 -19.457 0.417 1.000 -ATOM 2248 OH2 TIP3 2587 5.100 -14.257 1.879 -0.834 1.520 -ATOM 2249 H1 TIP3 2587 5.444 -14.912 1.185 0.417 1.000 -ATOM 2250 H2 TIP3 2587 5.721 -14.424 2.596 0.417 1.000 -ATOM 2251 OH2 TIP3 2591 -10.485 -19.389 10.327 -0.834 1.520 -ATOM 2252 H1 TIP3 2591 -9.993 -19.757 9.596 0.417 1.000 -ATOM 2253 H2 TIP3 2591 -9.690 -18.964 10.864 0.417 1.000 -ATOM 2254 OH2 TIP3 2606 -1.673 -19.625 -8.542 -0.834 1.520 -ATOM 2255 H1 TIP3 2606 -1.175 -19.859 -7.706 0.417 1.000 -ATOM 2256 H2 TIP3 2606 -1.597 -18.634 -8.620 0.417 1.000 -ATOM 2257 OH2 TIP3 2607 -9.950 -14.395 -14.242 -0.834 1.520 -ATOM 2258 H1 TIP3 2607 -9.845 -13.446 -14.444 0.417 1.000 -ATOM 2259 H2 TIP3 2607 -9.203 -14.611 -13.691 0.417 1.000 -ATOM 2260 OH2 TIP3 2608 -12.119 -16.482 -14.300 -0.834 1.520 -ATOM 2261 H1 TIP3 2608 -11.450 -15.775 -14.188 0.417 1.000 -ATOM 2262 H2 TIP3 2608 -12.930 -16.166 -13.831 0.417 1.000 -ATOM 2263 OH2 TIP3 2610 5.589 -14.730 -7.463 -0.834 1.520 -ATOM 2264 H1 TIP3 2610 5.815 -15.522 -7.882 0.417 1.000 -ATOM 2265 H2 TIP3 2610 6.224 -14.566 -6.743 0.417 1.000 -ATOM 2266 OH2 TIP3 2616 -5.672 -6.663 10.714 -0.834 1.520 -ATOM 2267 H1 TIP3 2616 -5.988 -5.859 10.228 0.417 1.000 -ATOM 2268 H2 TIP3 2616 -5.774 -6.445 11.617 0.417 1.000 -ATOM 2269 OH2 TIP3 2617 -1.197 -14.024 11.699 -0.834 1.520 -ATOM 2270 H1 TIP3 2617 -0.751 -14.772 12.081 0.417 1.000 -ATOM 2271 H2 TIP3 2617 -0.382 -13.434 11.513 0.417 1.000 -ATOM 2272 OH2 TIP3 2628 6.162 -10.473 -9.944 -0.834 1.520 -ATOM 2273 H1 TIP3 2628 5.979 -11.367 -9.666 0.417 1.000 -ATOM 2274 H2 TIP3 2628 5.294 -9.977 -9.718 0.417 1.000 -ATOM 2275 OH2 TIP3 2632 5.214 -0.932 -4.568 -0.834 1.520 -ATOM 2276 H1 TIP3 2632 4.652 -1.270 -3.860 0.417 1.000 -ATOM 2277 H2 TIP3 2632 4.602 -0.856 -5.342 0.417 1.000 -ATOM 2278 OH2 TIP3 2633 3.139 -3.798 8.164 -0.834 1.520 -ATOM 2279 H1 TIP3 2633 3.293 -2.858 8.197 0.417 1.000 -ATOM 2280 H2 TIP3 2633 3.953 -4.145 8.642 0.417 1.000 -ATOM 2281 OH2 TIP3 2634 -11.441 -16.485 -8.465 -0.834 1.520 -ATOM 2282 H1 TIP3 2634 -11.565 -15.702 -7.845 0.417 1.000 -ATOM 2283 H2 TIP3 2634 -12.045 -17.101 -8.048 0.417 1.000 -ATOM 2284 OH2 TIP3 2635 13.234 -18.643 1.754 -0.834 1.520 -ATOM 2285 H1 TIP3 2635 13.961 -18.998 1.291 0.417 1.000 -ATOM 2286 H2 TIP3 2635 12.849 -18.048 1.080 0.417 1.000 -ATOM 2287 OH2 TIP3 2649 9.773 -10.375 -10.236 -0.834 1.520 -ATOM 2288 H1 TIP3 2649 9.262 -10.746 -9.526 0.417 1.000 -ATOM 2289 H2 TIP3 2649 9.260 -9.559 -10.423 0.417 1.000 -ATOM 2290 OH2 TIP3 2650 5.669 -16.299 -10.966 -0.834 1.520 -ATOM 2291 H1 TIP3 2650 5.837 -17.235 -11.111 0.417 1.000 -ATOM 2292 H2 TIP3 2650 6.139 -16.085 -10.145 0.417 1.000 -ATOM 2293 OH2 TIP3 2654 7.204 -10.323 -12.424 -0.834 1.520 -ATOM 2294 H1 TIP3 2654 7.643 -9.604 -11.980 0.417 1.000 -ATOM 2295 H2 TIP3 2654 6.766 -10.747 -11.674 0.417 1.000 -ATOM 2296 OH2 TIP3 2656 -13.968 -3.657 6.922 -0.834 1.520 -ATOM 2297 H1 TIP3 2656 -14.881 -3.831 6.946 0.417 1.000 -ATOM 2298 H2 TIP3 2656 -13.889 -3.435 5.983 0.417 1.000 -ATOM 2299 OH2 TIP3 2657 -3.286 -14.989 14.329 -0.834 1.520 -ATOM 2300 H1 TIP3 2657 -3.307 -15.911 14.500 0.417 1.000 -ATOM 2301 H2 TIP3 2657 -2.869 -15.000 13.463 0.417 1.000 -ATOM 2302 OH2 TIP3 2659 -3.076 -11.014 17.568 -0.834 1.520 -ATOM 2303 H1 TIP3 2659 -2.482 -11.367 18.338 0.417 1.000 -ATOM 2304 H2 TIP3 2659 -3.165 -11.780 16.959 0.417 1.000 -ATOM 2305 OH2 TIP3 2662 0.109 -14.403 19.779 -0.834 1.520 -ATOM 2306 H1 TIP3 2662 1.070 -14.633 19.700 0.417 1.000 -ATOM 2307 H2 TIP3 2662 -0.245 -14.861 19.051 0.417 1.000 -ATOM 2308 OH2 TIP3 2667 12.951 -14.346 -0.591 -0.834 1.520 -ATOM 2309 H1 TIP3 2667 12.204 -14.063 -0.012 0.417 1.000 -ATOM 2310 H2 TIP3 2667 12.732 -14.152 -1.551 0.417 1.000 -ATOM 2311 OH2 TIP3 2669 -6.961 -9.489 -8.223 -0.834 1.520 -ATOM 2312 H1 TIP3 2669 -6.175 -9.411 -8.718 0.417 1.000 -ATOM 2313 H2 TIP3 2669 -7.117 -8.527 -8.084 0.417 1.000 -ATOM 2314 OH2 TIP3 2670 7.684 -10.298 -3.912 -0.834 1.520 -ATOM 2315 H1 TIP3 2670 7.282 -11.211 -3.787 0.417 1.000 -ATOM 2316 H2 TIP3 2670 7.247 -9.756 -3.229 0.417 1.000 -ATOM 2317 OH2 TIP3 2672 15.298 -9.010 -3.999 -0.834 1.520 -ATOM 2318 H1 TIP3 2672 15.295 -8.277 -4.546 0.417 1.000 -ATOM 2319 H2 TIP3 2672 14.284 -9.261 -3.912 0.417 1.000 -ATOM 2320 OH2 TIP3 2673 -1.811 -15.418 -1.151 -0.834 1.520 -ATOM 2321 H1 TIP3 2673 -2.408 -15.228 -0.409 0.417 1.000 -ATOM 2322 H2 TIP3 2673 -1.538 -14.571 -1.482 0.417 1.000 -ATOM 2323 OH2 TIP3 2674 13.936 -18.170 -2.122 -0.834 1.520 -ATOM 2324 H1 TIP3 2674 13.988 -17.572 -2.901 0.417 1.000 -ATOM 2325 H2 TIP3 2674 13.180 -18.802 -2.269 0.417 1.000 -ATOM 2326 OH2 TIP3 2675 -6.413 -18.148 6.990 -0.834 1.520 -ATOM 2327 H1 TIP3 2675 -5.889 -18.229 6.142 0.417 1.000 -ATOM 2328 H2 TIP3 2675 -6.304 -19.035 7.380 0.417 1.000 -ATOM 2329 OH2 TIP3 2676 5.774 -16.116 -0.004 -0.834 1.520 -ATOM 2330 H1 TIP3 2676 5.595 -17.004 0.380 0.417 1.000 -ATOM 2331 H2 TIP3 2676 5.873 -16.363 -0.998 0.417 1.000 -ATOM 2332 OH2 TIP3 2677 -8.740 -6.761 5.511 -0.834 1.520 -ATOM 2333 H1 TIP3 2677 -8.935 -5.999 5.990 0.417 1.000 -ATOM 2334 H2 TIP3 2677 -8.347 -7.410 6.175 0.417 1.000 -ATOM 2335 OH2 TIP3 2678 8.025 -6.221 9.088 -0.834 1.520 -ATOM 2336 H1 TIP3 2678 8.704 -5.819 8.609 0.417 1.000 -ATOM 2337 H2 TIP3 2678 7.189 -5.812 8.743 0.417 1.000 -ATOM 2338 OH2 TIP3 2680 0.331 -12.429 15.970 -0.834 1.520 -ATOM 2339 H1 TIP3 2680 -0.596 -12.261 15.737 0.417 1.000 -ATOM 2340 H2 TIP3 2680 0.797 -12.749 15.171 0.417 1.000 -ATOM 2341 OH2 TIP3 2683 0.059 -19.029 15.754 -0.834 1.520 -ATOM 2342 H1 TIP3 2683 -0.305 -19.160 16.633 0.417 1.000 -ATOM 2343 H2 TIP3 2683 -0.545 -19.575 15.252 0.417 1.000 -ATOM 2344 OH2 TIP3 2689 1.509 -15.070 -13.679 -0.834 1.520 -ATOM 2345 H1 TIP3 2689 1.670 -14.168 -13.722 0.417 1.000 -ATOM 2346 H2 TIP3 2689 2.198 -15.413 -13.036 0.417 1.000 -ATOM 2347 OH2 TIP3 2690 3.213 -14.759 -11.172 -0.834 1.520 -ATOM 2348 H1 TIP3 2690 4.050 -15.326 -10.885 0.417 1.000 -ATOM 2349 H2 TIP3 2690 3.460 -13.914 -10.826 0.417 1.000 -ATOM 2350 OH2 TIP3 2691 -4.684 -16.024 -4.967 -0.834 1.520 -ATOM 2351 H1 TIP3 2691 -4.647 -15.735 -5.935 0.417 1.000 -ATOM 2352 H2 TIP3 2691 -5.415 -15.467 -4.540 0.417 1.000 -ATOM 2353 OH2 TIP3 2692 5.596 -13.267 -9.669 -0.834 1.520 -ATOM 2354 H1 TIP3 2692 6.356 -13.859 -9.906 0.417 1.000 -ATOM 2355 H2 TIP3 2692 5.494 -13.449 -8.709 0.417 1.000 -ATOM 2356 OH2 TIP3 2693 -2.949 -18.510 10.533 -0.834 1.520 -ATOM 2357 H1 TIP3 2693 -2.514 -18.280 9.668 0.417 1.000 -ATOM 2358 H2 TIP3 2693 -3.722 -19.073 10.237 0.417 1.000 -ATOM 2359 OH2 TIP3 2694 -11.600 -14.448 -6.443 -0.834 1.520 -ATOM 2360 H1 TIP3 2694 -10.930 -14.760 -5.815 0.417 1.000 -ATOM 2361 H2 TIP3 2694 -12.261 -14.055 -5.837 0.417 1.000 -ATOM 2362 OH2 TIP3 2695 13.084 -11.238 -5.690 -0.834 1.520 -ATOM 2363 H1 TIP3 2695 12.733 -10.456 -5.249 0.417 1.000 -ATOM 2364 H2 TIP3 2695 12.272 -11.743 -5.877 0.417 1.000 -ATOM 2365 OH2 TIP3 2696 12.654 -16.309 4.596 -0.834 1.520 -ATOM 2366 H1 TIP3 2696 12.880 -16.116 5.513 0.417 1.000 -ATOM 2367 H2 TIP3 2696 12.744 -17.242 4.501 0.417 1.000 -ATOM 2368 OH2 TIP3 2697 3.742 -4.291 -1.904 -0.834 1.520 -ATOM 2369 H1 TIP3 2697 4.551 -3.894 -2.179 0.417 1.000 -ATOM 2370 H2 TIP3 2697 3.413 -4.712 -2.769 0.417 1.000 -ATOM 2371 OH2 TIP3 2699 8.004 -2.354 8.912 -0.834 1.520 -ATOM 2372 H1 TIP3 2699 7.434 -1.589 9.281 0.417 1.000 -ATOM 2373 H2 TIP3 2699 7.354 -2.956 8.559 0.417 1.000 -ATOM 2374 OH2 TIP3 2701 -8.206 5.052 -2.354 -0.834 1.520 -ATOM 2375 H1 TIP3 2701 -7.333 4.804 -2.716 0.417 1.000 -ATOM 2376 H2 TIP3 2701 -8.394 4.326 -1.718 0.417 1.000 -ATOM 2377 OH2 TIP3 2703 4.549 -12.992 15.738 -0.834 1.520 -ATOM 2378 H1 TIP3 2703 3.797 -13.555 15.329 0.417 1.000 -ATOM 2379 H2 TIP3 2703 4.671 -13.424 16.621 0.417 1.000 -ATOM 2380 OH2 TIP3 2709 -0.746 1.282 -17.342 -0.834 1.520 -ATOM 2381 H1 TIP3 2709 -0.081 0.761 -16.859 0.417 1.000 -ATOM 2382 H2 TIP3 2709 -1.319 0.629 -17.676 0.417 1.000 -ATOM 2383 OH2 TIP3 2710 11.648 -3.762 -18.762 -0.834 1.520 -ATOM 2384 H1 TIP3 2710 11.394 -2.870 -18.371 0.417 1.000 -ATOM 2385 H2 TIP3 2710 10.936 -4.357 -18.482 0.417 1.000 -ATOM 2386 OH2 TIP3 2713 -3.886 0.023 -11.152 -0.834 1.520 -ATOM 2387 H1 TIP3 2713 -3.768 0.963 -11.391 0.417 1.000 -ATOM 2388 H2 TIP3 2713 -4.743 -0.085 -10.700 0.417 1.000 -ATOM 2389 OH2 TIP3 2714 -15.719 1.161 -0.934 -0.834 1.520 -ATOM 2390 H1 TIP3 2714 -14.843 1.058 -0.619 0.417 1.000 -ATOM 2391 H2 TIP3 2714 -15.831 2.110 -1.096 0.417 1.000 -ATOM 2392 OH2 TIP3 2715 1.505 -11.802 -13.855 -0.834 1.520 -ATOM 2393 H1 TIP3 2715 1.809 -10.929 -13.681 0.417 1.000 -ATOM 2394 H2 TIP3 2715 1.328 -11.751 -14.803 0.417 1.000 -ATOM 2395 OH2 TIP3 2717 11.568 1.320 1.163 -0.834 1.520 -ATOM 2396 H1 TIP3 2717 10.799 0.711 1.305 0.417 1.000 -ATOM 2397 H2 TIP3 2717 11.323 1.603 0.269 0.417 1.000 -ATOM 2398 OH2 TIP3 2718 8.597 2.409 18.875 -0.834 1.520 -ATOM 2399 H1 TIP3 2718 8.089 2.419 19.687 0.417 1.000 -ATOM 2400 H2 TIP3 2718 8.701 1.435 18.799 0.417 1.000 -ATOM 2401 OH2 TIP3 2719 5.995 -7.637 5.902 -0.834 1.520 -ATOM 2402 H1 TIP3 2719 5.653 -6.793 5.530 0.417 1.000 -ATOM 2403 H2 TIP3 2719 5.203 -8.194 5.809 0.417 1.000 -ATOM 2404 OH2 TIP3 2720 -2.456 -12.747 15.428 -0.834 1.520 -ATOM 2405 H1 TIP3 2720 -2.917 -12.083 14.937 0.417 1.000 -ATOM 2406 H2 TIP3 2720 -2.819 -13.576 15.037 0.417 1.000 -ATOM 2407 OH2 TIP3 2721 8.586 -8.743 16.679 -0.834 1.520 -ATOM 2408 H1 TIP3 2721 9.542 -8.629 16.997 0.417 1.000 -ATOM 2409 H2 TIP3 2721 8.692 -9.379 15.967 0.417 1.000 -ATOM 2410 OH2 TIP3 2732 -6.130 -12.526 -7.813 -0.834 1.520 -ATOM 2411 H1 TIP3 2732 -6.791 -11.927 -7.367 0.417 1.000 -ATOM 2412 H2 TIP3 2732 -5.543 -11.971 -8.341 0.417 1.000 -ATOM 2413 OH2 TIP3 2734 0.488 -6.827 -13.356 -0.834 1.520 -ATOM 2414 H1 TIP3 2734 -0.366 -7.303 -13.069 0.417 1.000 -ATOM 2415 H2 TIP3 2734 0.688 -6.266 -12.550 0.417 1.000 -ATOM 2416 OH2 TIP3 2735 -6.400 -2.753 -10.002 -0.834 1.520 -ATOM 2417 H1 TIP3 2735 -6.744 -3.615 -9.893 0.417 1.000 -ATOM 2418 H2 TIP3 2735 -6.384 -2.663 -10.953 0.417 1.000 -ATOM 2419 OH2 TIP3 2736 9.201 -8.366 -12.246 -0.834 1.520 -ATOM 2420 H1 TIP3 2736 9.874 -7.686 -12.123 0.417 1.000 -ATOM 2421 H2 TIP3 2736 9.387 -8.653 -13.190 0.417 1.000 -ATOM 2422 OH2 TIP3 2737 3.968 -5.752 5.301 -0.834 1.520 -ATOM 2423 H1 TIP3 2737 3.469 -5.336 6.014 0.417 1.000 -ATOM 2424 H2 TIP3 2737 3.898 -5.141 4.541 0.417 1.000 -ATOM 2425 OH2 TIP3 2738 -10.766 6.832 5.736 -0.834 1.520 -ATOM 2426 H1 TIP3 2738 -10.254 6.180 6.234 0.417 1.000 -ATOM 2427 H2 TIP3 2738 -11.116 6.235 5.066 0.417 1.000 -ATOM 2428 OH2 TIP3 2739 14.024 7.737 5.007 -0.834 1.520 -ATOM 2429 H1 TIP3 2739 14.615 7.021 5.086 0.417 1.000 -ATOM 2430 H2 TIP3 2739 14.589 8.553 4.850 0.417 1.000 -ATOM 2431 OH2 TIP3 2740 7.087 -0.388 3.133 -0.834 1.520 -ATOM 2432 H1 TIP3 2740 7.094 -1.103 3.801 0.417 1.000 -ATOM 2433 H2 TIP3 2740 6.150 -0.210 3.002 0.417 1.000 -ATOM 2434 OH2 TIP3 2741 -3.495 2.219 11.618 -0.834 1.520 -ATOM 2435 H1 TIP3 2741 -2.750 1.631 11.854 0.417 1.000 -ATOM 2436 H2 TIP3 2741 -2.987 2.977 11.275 0.417 1.000 -ATOM 2437 OH2 TIP3 2743 -1.524 -6.983 6.155 -0.834 1.520 -ATOM 2438 H1 TIP3 2743 -2.124 -7.442 5.566 0.417 1.000 -ATOM 2439 H2 TIP3 2743 -1.643 -7.392 7.053 0.417 1.000 -ATOM 2440 OH2 TIP3 2753 0.306 6.975 -13.863 -0.834 1.520 -ATOM 2441 H1 TIP3 2753 -0.571 7.310 -13.743 0.417 1.000 -ATOM 2442 H2 TIP3 2753 0.838 7.794 -13.981 0.417 1.000 -ATOM 2443 OH2 TIP3 2754 4.900 -9.096 -6.137 -0.834 1.520 -ATOM 2444 H1 TIP3 2754 4.395 -8.989 -6.964 0.417 1.000 -ATOM 2445 H2 TIP3 2754 5.838 -9.213 -6.379 0.417 1.000 -ATOM 2446 OH2 TIP3 2755 -13.352 6.409 -0.992 -0.834 1.520 -ATOM 2447 H1 TIP3 2755 -12.956 7.167 -0.499 0.417 1.000 -ATOM 2448 H2 TIP3 2755 -13.010 6.530 -1.849 0.417 1.000 -ATOM 2449 OH2 TIP3 2756 8.512 -8.004 6.333 -0.834 1.520 -ATOM 2450 H1 TIP3 2756 9.099 -7.178 6.113 0.417 1.000 -ATOM 2451 H2 TIP3 2756 7.537 -7.755 6.168 0.417 1.000 -ATOM 2452 OH2 TIP3 2757 -3.154 5.884 -2.277 -0.834 1.520 -ATOM 2453 H1 TIP3 2757 -3.457 5.668 -3.194 0.417 1.000 -ATOM 2454 H2 TIP3 2757 -3.531 6.717 -2.238 0.417 1.000 -ATOM 2455 OH2 TIP3 2758 -9.901 -1.917 1.099 -0.834 1.520 -ATOM 2456 H1 TIP3 2758 -9.547 -2.742 0.699 0.417 1.000 -ATOM 2457 H2 TIP3 2758 -10.602 -2.254 1.690 0.417 1.000 -ATOM 2458 OH2 TIP3 2760 -6.517 -7.344 7.905 -0.834 1.520 -ATOM 2459 H1 TIP3 2760 -6.435 -7.079 8.822 0.417 1.000 -ATOM 2460 H2 TIP3 2760 -6.042 -8.201 7.830 0.417 1.000 -ATOM 2461 OH2 TIP3 2761 16.470 -11.691 14.969 -0.834 1.520 -ATOM 2462 H1 TIP3 2761 16.184 -11.645 13.997 0.417 1.000 -ATOM 2463 H2 TIP3 2761 17.427 -11.531 14.971 0.417 1.000 -ATOM 2464 OH2 TIP3 2767 -13.647 12.350 19.261 -0.834 1.520 -ATOM 2465 H1 TIP3 2767 -13.550 11.938 18.415 0.417 1.000 -ATOM 2466 H2 TIP3 2767 -14.608 12.146 19.419 0.417 1.000 -ATOM 2467 OH2 TIP3 2771 3.915 -3.797 -15.779 -0.834 1.520 -ATOM 2468 H1 TIP3 2771 4.358 -4.064 -14.935 0.417 1.000 -ATOM 2469 H2 TIP3 2771 3.944 -4.654 -16.232 0.417 1.000 -ATOM 2470 OH2 TIP3 2774 13.812 9.049 -8.862 -0.834 1.520 -ATOM 2471 H1 TIP3 2774 13.368 8.234 -9.056 0.417 1.000 -ATOM 2472 H2 TIP3 2774 14.731 8.861 -9.092 0.417 1.000 -ATOM 2473 OH2 TIP3 2775 9.677 -11.246 -15.552 -0.834 1.520 -ATOM 2474 H1 TIP3 2775 9.019 -10.574 -15.342 0.417 1.000 -ATOM 2475 H2 TIP3 2775 9.888 -11.607 -14.699 0.417 1.000 -ATOM 2476 OH2 TIP3 2776 -2.398 -0.378 1.532 -0.834 1.520 -ATOM 2477 H1 TIP3 2776 -1.724 -0.621 2.185 0.417 1.000 -ATOM 2478 H2 TIP3 2776 -2.876 -1.189 1.522 0.417 1.000 -ATOM 2479 OH2 TIP3 2777 6.417 0.249 -12.088 -0.834 1.520 -ATOM 2480 H1 TIP3 2777 5.862 0.922 -12.459 0.417 1.000 -ATOM 2481 H2 TIP3 2777 7.149 0.907 -11.751 0.417 1.000 -ATOM 2482 OH2 TIP3 2778 -4.535 -12.045 -2.186 -0.834 1.520 -ATOM 2483 H1 TIP3 2778 -4.850 -12.021 -3.068 0.417 1.000 -ATOM 2484 H2 TIP3 2778 -5.294 -12.437 -1.694 0.417 1.000 -ATOM 2485 OH2 TIP3 2779 -10.844 -7.686 3.966 -0.834 1.520 -ATOM 2486 H1 TIP3 2779 -11.134 -8.115 4.871 0.417 1.000 -ATOM 2487 H2 TIP3 2779 -9.914 -7.519 4.093 0.417 1.000 -ATOM 2488 OH2 TIP3 2780 -15.853 4.095 8.652 -0.834 1.520 -ATOM 2489 H1 TIP3 2780 -15.475 4.949 8.960 0.417 1.000 -ATOM 2490 H2 TIP3 2780 -16.667 4.040 9.113 0.417 1.000 -ATOM 2491 OH2 TIP3 2781 -18.093 18.378 -9.838 -0.834 1.520 -ATOM 2492 H1 TIP3 2781 -17.693 17.490 -9.732 0.417 1.000 -ATOM 2493 H2 TIP3 2781 -18.252 18.649 -8.926 0.417 1.000 -ATOM 2494 OH2 TIP3 2782 12.884 8.064 10.649 -0.834 1.520 -ATOM 2495 H1 TIP3 2782 13.631 8.473 11.131 0.417 1.000 -ATOM 2496 H2 TIP3 2782 13.301 7.781 9.866 0.417 1.000 -ATOM 2497 OH2 TIP3 2783 9.419 6.542 19.110 -0.834 1.520 -ATOM 2498 H1 TIP3 2783 8.963 6.568 18.230 0.417 1.000 -ATOM 2499 H2 TIP3 2783 10.285 6.854 18.907 0.417 1.000 -ATOM 2500 OH2 TIP3 2784 -7.728 9.826 12.189 -0.834 1.520 -ATOM 2501 H1 TIP3 2784 -7.844 10.869 12.133 0.417 1.000 -ATOM 2502 H2 TIP3 2784 -7.894 9.537 11.318 0.417 1.000 -ATOM 2503 OH2 TIP3 2794 2.042 -9.231 -13.524 -0.834 1.520 -ATOM 2504 H1 TIP3 2794 2.762 -8.834 -12.936 0.417 1.000 -ATOM 2505 H2 TIP3 2794 1.379 -8.499 -13.654 0.417 1.000 -ATOM 2506 OH2 TIP3 2795 -4.419 8.901 -11.059 -0.834 1.520 -ATOM 2507 H1 TIP3 2795 -4.125 9.699 -11.454 0.417 1.000 -ATOM 2508 H2 TIP3 2795 -3.623 8.333 -11.092 0.417 1.000 -ATOM 2509 OH2 TIP3 2797 3.031 9.489 -4.238 -0.834 1.520 -ATOM 2510 H1 TIP3 2797 3.931 9.176 -4.113 0.417 1.000 -ATOM 2511 H2 TIP3 2797 2.945 9.581 -5.203 0.417 1.000 -ATOM 2512 OH2 TIP3 2799 4.053 5.323 16.555 -0.834 1.520 -ATOM 2513 H1 TIP3 2799 3.778 4.765 17.255 0.417 1.000 -ATOM 2514 H2 TIP3 2799 3.538 5.083 15.793 0.417 1.000 -ATOM 2515 OH2 TIP3 2800 -0.963 -0.357 10.882 -0.834 1.520 -ATOM 2516 H1 TIP3 2800 -1.558 -1.117 11.027 0.417 1.000 -ATOM 2517 H2 TIP3 2800 -0.818 0.037 11.784 0.417 1.000 -ATOM 2518 OH2 TIP3 2801 -3.207 -9.793 9.783 -0.834 1.520 -ATOM 2519 H1 TIP3 2801 -3.388 -10.456 10.483 0.417 1.000 -ATOM 2520 H2 TIP3 2801 -2.292 -10.035 9.460 0.417 1.000 -ATOM 2521 OH2 TIP3 2802 -17.239 14.861 -2.664 -0.834 1.520 -ATOM 2522 H1 TIP3 2802 -17.849 14.338 -2.118 0.417 1.000 -ATOM 2523 H2 TIP3 2802 -17.352 14.491 -3.501 0.417 1.000 -ATOM 2524 OH2 TIP3 2803 5.886 15.235 18.227 -0.834 1.520 -ATOM 2525 H1 TIP3 2803 5.644 14.327 18.100 0.417 1.000 -ATOM 2526 H2 TIP3 2803 6.487 15.306 19.006 0.417 1.000 -ATOM 2527 OH2 TIP3 2806 -8.534 7.575 7.795 -0.834 1.520 -ATOM 2528 H1 TIP3 2806 -8.324 7.642 6.825 0.417 1.000 -ATOM 2529 H2 TIP3 2806 -8.909 6.734 7.835 0.417 1.000 -ATOM 2530 OH2 TIP3 2815 0.395 -10.430 -8.366 -0.834 1.520 -ATOM 2531 H1 TIP3 2815 1.018 -11.111 -8.144 0.417 1.000 -ATOM 2532 H2 TIP3 2815 0.725 -9.687 -7.898 0.417 1.000 -ATOM 2533 OH2 TIP3 2817 -0.108 3.014 -1.949 -0.834 1.520 -ATOM 2534 H1 TIP3 2817 0.854 3.005 -1.812 0.417 1.000 -ATOM 2535 H2 TIP3 2817 -0.299 4.011 -2.047 0.417 1.000 -ATOM 2536 OH2 TIP3 2818 4.927 4.245 -4.003 -0.834 1.520 -ATOM 2537 H1 TIP3 2818 5.786 4.617 -4.065 0.417 1.000 -ATOM 2538 H2 TIP3 2818 4.853 4.047 -3.021 0.417 1.000 -ATOM 2539 OH2 TIP3 2819 4.700 13.170 -9.255 -0.834 1.520 -ATOM 2540 H1 TIP3 2819 5.184 12.434 -9.631 0.417 1.000 -ATOM 2541 H2 TIP3 2819 5.355 13.588 -8.658 0.417 1.000 -ATOM 2542 OH2 TIP3 2821 7.254 1.261 0.770 -0.834 1.520 -ATOM 2543 H1 TIP3 2821 7.904 1.713 1.261 0.417 1.000 -ATOM 2544 H2 TIP3 2821 6.810 0.778 1.494 0.417 1.000 -ATOM 2545 OH2 TIP3 2823 17.156 18.784 0.387 -0.834 1.520 -ATOM 2546 H1 TIP3 2823 16.616 18.131 -0.043 0.417 1.000 -ATOM 2547 H2 TIP3 2823 17.603 19.169 -0.403 0.417 1.000 -ATOM 2548 OH2 TIP3 2824 -11.747 -5.917 14.818 -0.834 1.520 -ATOM 2549 H1 TIP3 2824 -12.671 -6.137 14.718 0.417 1.000 -ATOM 2550 H2 TIP3 2824 -11.714 -5.329 15.651 0.417 1.000 -ATOM 2551 OH2 TIP3 2825 9.678 10.880 14.427 -0.834 1.520 -ATOM 2552 H1 TIP3 2825 10.466 10.742 13.888 0.417 1.000 -ATOM 2553 H2 TIP3 2825 10.062 11.228 15.205 0.417 1.000 -ATOM 2554 OH2 TIP3 2827 -6.023 2.652 13.422 -0.834 1.520 -ATOM 2555 H1 TIP3 2827 -5.241 2.740 12.797 0.417 1.000 -ATOM 2556 H2 TIP3 2827 -5.648 2.123 14.190 0.417 1.000 -ATOM 2557 OH2 TIP3 2832 -0.083 6.551 -10.960 -0.834 1.520 -ATOM 2558 H1 TIP3 2832 0.120 6.190 -11.860 0.417 1.000 -ATOM 2559 H2 TIP3 2832 -0.194 7.508 -11.129 0.417 1.000 -ATOM 2560 OH2 TIP3 2839 5.461 17.158 -4.472 -0.834 1.520 -ATOM 2561 H1 TIP3 2839 4.943 16.639 -5.094 0.417 1.000 -ATOM 2562 H2 TIP3 2839 4.876 17.217 -3.746 0.417 1.000 -ATOM 2563 OH2 TIP3 2840 -2.107 17.193 -1.716 -0.834 1.520 -ATOM 2564 H1 TIP3 2840 -2.302 17.733 -2.532 0.417 1.000 -ATOM 2565 H2 TIP3 2840 -1.831 17.859 -1.108 0.417 1.000 -ATOM 2566 OH2 TIP3 2841 1.057 13.054 -1.300 -0.834 1.520 -ATOM 2567 H1 TIP3 2841 1.397 13.374 -0.462 0.417 1.000 -ATOM 2568 H2 TIP3 2841 1.324 13.836 -1.915 0.417 1.000 -ATOM 2569 OH2 TIP3 2842 0.002 10.235 -5.227 -0.834 1.520 -ATOM 2570 H1 TIP3 2842 -0.320 10.758 -4.422 0.417 1.000 -ATOM 2571 H2 TIP3 2842 0.498 9.580 -4.815 0.417 1.000 -ATOM 2572 OH2 TIP3 2843 10.918 6.815 -3.533 -0.834 1.520 -ATOM 2573 H1 TIP3 2843 11.359 6.038 -3.268 0.417 1.000 -ATOM 2574 H2 TIP3 2843 10.366 7.062 -2.724 0.417 1.000 -ATOM 2575 OH2 TIP3 2844 -12.052 8.004 17.445 -0.834 1.520 -ATOM 2576 H1 TIP3 2844 -12.114 7.389 16.693 0.417 1.000 -ATOM 2577 H2 TIP3 2844 -11.385 8.635 17.269 0.417 1.000 -ATOM 2578 OH2 TIP3 2845 -5.998 2.548 7.246 -0.834 1.520 -ATOM 2579 H1 TIP3 2845 -6.001 2.223 6.330 0.417 1.000 -ATOM 2580 H2 TIP3 2845 -6.753 1.983 7.672 0.417 1.000 -ATOM 2581 OH2 TIP3 2848 2.534 18.000 15.364 -0.834 1.520 -ATOM 2582 H1 TIP3 2848 1.544 17.950 15.278 0.417 1.000 -ATOM 2583 H2 TIP3 2848 2.680 18.710 16.030 0.417 1.000 -ATOM 2584 OH2 TIP3 2857 -4.273 6.799 -8.529 -0.834 1.520 -ATOM 2585 H1 TIP3 2857 -3.553 6.161 -8.206 0.417 1.000 -ATOM 2586 H2 TIP3 2857 -4.332 7.359 -7.784 0.417 1.000 -ATOM 2587 OH2 TIP3 2859 -6.476 11.510 -15.250 -0.834 1.520 -ATOM 2588 H1 TIP3 2859 -5.698 12.013 -15.570 0.417 1.000 -ATOM 2589 H2 TIP3 2859 -7.210 11.810 -15.738 0.417 1.000 -ATOM 2590 OH2 TIP3 2860 1.077 14.629 7.395 -0.834 1.520 -ATOM 2591 H1 TIP3 2860 1.048 15.379 6.771 0.417 1.000 -ATOM 2592 H2 TIP3 2860 0.459 13.944 6.930 0.417 1.000 -ATOM 2593 OH2 TIP3 2862 -1.571 10.247 -0.425 -0.834 1.520 -ATOM 2594 H1 TIP3 2862 -2.321 9.554 -0.532 0.417 1.000 -ATOM 2595 H2 TIP3 2862 -2.021 10.898 0.090 0.417 1.000 -ATOM 2596 OH2 TIP3 2863 -5.861 14.491 -0.746 -0.834 1.520 -ATOM 2597 H1 TIP3 2863 -6.746 14.086 -0.607 0.417 1.000 -ATOM 2598 H2 TIP3 2863 -5.344 13.811 -1.252 0.417 1.000 -ATOM 2599 OH2 TIP3 2865 -5.011 10.052 7.685 -0.834 1.520 -ATOM 2600 H1 TIP3 2865 -4.914 9.323 7.155 0.417 1.000 -ATOM 2601 H2 TIP3 2865 -4.434 9.840 8.381 0.417 1.000 -ATOM 2602 OH2 TIP3 2866 -2.693 1.454 4.794 -0.834 1.520 -ATOM 2603 H1 TIP3 2866 -2.006 0.811 4.602 0.417 1.000 -ATOM 2604 H2 TIP3 2866 -2.220 2.132 5.344 0.417 1.000 -ATOM 2605 OH2 TIP3 2867 1.674 15.212 10.136 -0.834 1.520 -ATOM 2606 H1 TIP3 2867 1.346 14.932 9.292 0.417 1.000 -ATOM 2607 H2 TIP3 2867 1.751 14.443 10.663 0.417 1.000 -ATOM 2608 OH2 TIP3 2877 4.857 8.631 -18.261 -0.834 1.520 -ATOM 2609 H1 TIP3 2877 3.986 8.916 -18.688 0.417 1.000 -ATOM 2610 H2 TIP3 2877 5.159 9.541 -17.886 0.417 1.000 -ATOM 2611 OH2 TIP3 2878 1.862 7.684 -18.306 -0.834 1.520 -ATOM 2612 H1 TIP3 2878 1.911 8.691 -18.287 0.417 1.000 -ATOM 2613 H2 TIP3 2878 0.977 7.455 -18.015 0.417 1.000 -ATOM 2614 OH2 TIP3 2882 10.376 16.016 -1.679 -0.834 1.520 -ATOM 2615 H1 TIP3 2882 10.542 15.187 -2.116 0.417 1.000 -ATOM 2616 H2 TIP3 2882 9.490 16.074 -1.908 0.417 1.000 -ATOM 2617 OH2 TIP3 2884 -2.186 16.110 -9.299 -0.834 1.520 -ATOM 2618 H1 TIP3 2884 -1.712 15.457 -8.770 0.417 1.000 -ATOM 2619 H2 TIP3 2884 -2.657 16.669 -8.647 0.417 1.000 -ATOM 2620 OH2 TIP3 2885 1.319 1.993 16.278 -0.834 1.520 -ATOM 2621 H1 TIP3 2885 1.626 1.109 15.812 0.417 1.000 -ATOM 2622 H2 TIP3 2885 2.130 2.551 16.370 0.417 1.000 -ATOM 2623 OH2 TIP3 2886 0.382 18.751 8.484 -0.834 1.520 -ATOM 2624 H1 TIP3 2886 0.560 18.177 7.677 0.417 1.000 -ATOM 2625 H2 TIP3 2886 1.200 19.321 8.548 0.417 1.000 -ATOM 2626 OH2 TIP3 2893 5.029 19.028 17.694 -0.834 1.520 -ATOM 2627 H1 TIP3 2893 5.498 18.942 18.542 0.417 1.000 -ATOM 2628 H2 TIP3 2893 4.146 18.871 17.933 0.417 1.000 -ATOM 2629 OH2 TIP3 2903 -14.445 11.449 -17.376 -0.834 1.520 -ATOM 2630 H1 TIP3 2903 -14.042 12.299 -17.283 0.417 1.000 -ATOM 2631 H2 TIP3 2903 -13.964 10.896 -16.785 0.417 1.000 -ATOM 2632 OH2 TIP3 2904 -1.957 16.132 1.546 -0.834 1.520 -ATOM 2633 H1 TIP3 2904 -1.744 16.487 0.682 0.417 1.000 -ATOM 2634 H2 TIP3 2904 -2.887 16.377 1.741 0.417 1.000 -ATOM 2635 OH2 TIP3 2906 -15.967 18.122 11.647 -0.834 1.520 -ATOM 2636 H1 TIP3 2906 -16.211 19.030 11.681 0.417 1.000 -ATOM 2637 H2 TIP3 2906 -15.388 18.170 10.894 0.417 1.000 -ATOM 2638 OH2 TIP3 2909 -5.592 18.995 16.722 -0.834 1.520 -ATOM 2639 H1 TIP3 2909 -6.204 19.707 16.506 0.417 1.000 -ATOM 2640 H2 TIP3 2909 -6.168 18.231 16.698 0.417 1.000 -ATOM 2641 OH2 TIP3 2918 -7.468 15.205 -16.074 -0.834 1.520 -ATOM 2642 H1 TIP3 2918 -6.681 15.020 -16.587 0.417 1.000 -ATOM 2643 H2 TIP3 2918 -7.436 16.141 -15.888 0.417 1.000 -ATOM 2644 OH2 TIP3 2919 3.513 12.168 -18.444 -0.834 1.520 -ATOM 2645 H1 TIP3 2919 2.857 11.774 -19.120 0.417 1.000 -ATOM 2646 H2 TIP3 2919 4.399 11.777 -18.696 0.417 1.000 -ATOM 2647 OH2 TIP3 2922 -8.486 12.235 -3.257 -0.834 1.520 -ATOM 2648 H1 TIP3 2922 -9.093 11.695 -3.750 0.417 1.000 -ATOM 2649 H2 TIP3 2922 -7.613 11.824 -3.274 0.417 1.000 -ATOM 2650 OH2 TIP3 2923 10.302 14.690 -7.785 -0.834 1.520 -ATOM 2651 H1 TIP3 2923 10.568 13.747 -7.852 0.417 1.000 -ATOM 2652 H2 TIP3 2923 9.350 14.665 -8.018 0.417 1.000 -ATOM 2653 OH2 TIP3 2925 -3.754 18.545 -8.100 -0.834 1.520 -ATOM 2654 H1 TIP3 2925 -4.529 18.979 -8.513 0.417 1.000 -ATOM 2655 H2 TIP3 2925 -3.381 19.257 -7.562 0.417 1.000 -ATOM 2656 OH2 TIP3 2926 -7.176 6.812 5.474 -0.834 1.520 -ATOM 2657 H1 TIP3 2926 -6.305 7.072 5.817 0.417 1.000 -ATOM 2658 H2 TIP3 2926 -7.100 5.883 5.335 0.417 1.000 -ATOM 2659 OH2 TIP3 2927 0.360 18.907 -10.832 -0.834 1.520 -ATOM 2660 H1 TIP3 2927 -0.330 18.507 -11.359 0.417 1.000 -ATOM 2661 H2 TIP3 2927 1.196 18.705 -11.355 0.417 1.000 -ATOM 2662 OH2 TIP3 2930 3.339 19.789 13.361 -0.834 1.520 -ATOM 2663 H1 TIP3 2930 4.082 19.549 12.752 0.417 1.000 -ATOM 2664 H2 TIP3 2930 3.115 18.890 13.780 0.417 1.000 -ATOM 2665 OH2 TIP3 2941 12.723 18.488 -6.535 -0.834 1.520 -ATOM 2666 H1 TIP3 2941 11.976 18.590 -7.127 0.417 1.000 -ATOM 2667 H2 TIP3 2941 13.229 17.813 -6.991 0.417 1.000 -ATOM 2668 OH2 TIP3 2944 5.815 10.983 -10.743 -0.834 1.520 -ATOM 2669 H1 TIP3 2944 6.628 11.032 -11.271 0.417 1.000 -ATOM 2670 H2 TIP3 2944 5.220 10.934 -11.527 0.417 1.000 -ATOM 2671 OH2 TIP3 2963 1.379 19.620 -13.468 -0.834 1.520 -ATOM 2672 H1 TIP3 2963 0.426 19.562 -13.395 0.417 1.000 -ATOM 2673 H2 TIP3 2963 1.498 19.479 -14.385 0.417 1.000 -ATOM 2674 OH2 TIP3 2968 6.037 16.386 4.824 -0.834 1.520 -ATOM 2675 H1 TIP3 2968 6.954 16.405 5.261 0.417 1.000 -ATOM 2676 H2 TIP3 2968 5.427 16.257 5.519 0.417 1.000 -ATOM 2677 OH2 TIP3 3010 -10.984 -15.914 -11.186 -0.834 1.520 -ATOM 2678 H1 TIP3 3010 -10.623 -14.999 -11.187 0.417 1.000 -ATOM 2679 H2 TIP3 3010 -11.274 -16.259 -10.339 0.417 1.000 -ATOM 2680 OH2 TIP3 3012 -0.545 -18.744 -3.183 -0.834 1.520 -ATOM 2681 H1 TIP3 3012 -0.024 -18.253 -3.745 0.417 1.000 -ATOM 2682 H2 TIP3 3012 -1.430 -18.359 -3.281 0.417 1.000 -ATOM 2683 OH2 TIP3 3016 -7.535 -15.131 6.264 -0.834 1.520 -ATOM 2684 H1 TIP3 3016 -7.270 -15.211 5.350 0.417 1.000 -ATOM 2685 H2 TIP3 3016 -6.778 -15.426 6.728 0.417 1.000 -ATOM 2686 OH2 TIP3 3027 -3.797 -17.798 0.733 -0.834 1.520 -ATOM 2687 H1 TIP3 3027 -4.277 -18.100 -0.057 0.417 1.000 -ATOM 2688 H2 TIP3 3027 -2.853 -17.956 0.493 0.417 1.000 -ATOM 2689 OH2 TIP3 3030 8.773 -10.723 3.205 -0.834 1.520 -ATOM 2690 H1 TIP3 3030 9.544 -11.354 3.079 0.417 1.000 -ATOM 2691 H2 TIP3 3030 9.043 -9.975 2.616 0.417 1.000 -ATOM 2692 OH2 TIP3 3033 2.922 -14.831 19.096 -0.834 1.520 -ATOM 2693 H1 TIP3 3033 3.219 -15.339 18.284 0.417 1.000 -ATOM 2694 H2 TIP3 3033 2.564 -14.047 18.614 0.417 1.000 -ATOM 2695 OH2 TIP3 3037 -1.660 -8.616 13.995 -0.834 1.520 -ATOM 2696 H1 TIP3 3037 -2.470 -8.801 13.453 0.417 1.000 -ATOM 2697 H2 TIP3 3037 -0.929 -9.112 13.559 0.417 1.000 -ATOM 2698 OH2 TIP3 3046 12.383 -11.977 -2.609 -0.834 1.520 -ATOM 2699 H1 TIP3 3046 11.933 -12.490 -3.377 0.417 1.000 -ATOM 2700 H2 TIP3 3046 11.666 -11.638 -2.081 0.417 1.000 -ATOM 2701 OH2 TIP3 3049 -1.047 -17.050 -6.856 -0.834 1.520 -ATOM 2702 H1 TIP3 3049 -1.091 -16.614 -5.981 0.417 1.000 -ATOM 2703 H2 TIP3 3049 -0.364 -17.660 -6.658 0.417 1.000 -ATOM 2704 OH2 TIP3 3050 10.894 -17.039 -4.656 -0.834 1.520 -ATOM 2705 H1 TIP3 3050 10.707 -17.698 -4.003 0.417 1.000 -ATOM 2706 H2 TIP3 3050 10.648 -17.474 -5.511 0.417 1.000 -ATOM 2707 OH2 TIP3 3053 -2.258 -18.096 7.902 -0.834 1.520 -ATOM 2708 H1 TIP3 3053 -1.627 -18.150 7.152 0.417 1.000 -ATOM 2709 H2 TIP3 3053 -3.002 -17.601 7.553 0.417 1.000 -ATOM 2710 OH2 TIP3 3070 -0.838 6.826 -6.278 -0.834 1.520 -ATOM 2711 H1 TIP3 3070 0.048 6.911 -6.581 0.417 1.000 -ATOM 2712 H2 TIP3 3070 -0.828 6.129 -5.645 0.417 1.000 -ATOM 2713 OH2 TIP3 3071 -3.247 -12.027 11.480 -0.834 1.520 -ATOM 2714 H1 TIP3 3071 -2.615 -12.719 11.626 0.417 1.000 -ATOM 2715 H2 TIP3 3071 -4.101 -12.443 11.384 0.417 1.000 -ATOM 2716 OH2 TIP3 3072 -13.561 -14.856 -12.545 -0.834 1.520 -ATOM 2717 H1 TIP3 3072 -12.690 -15.164 -12.163 0.417 1.000 -ATOM 2718 H2 TIP3 3072 -13.879 -14.181 -11.884 0.417 1.000 -ATOM 2719 OH2 TIP3 3073 -1.349 -15.351 -11.579 -0.834 1.520 -ATOM 2720 H1 TIP3 3073 -0.674 -14.678 -11.432 0.417 1.000 -ATOM 2721 H2 TIP3 3073 -1.637 -15.256 -12.507 0.417 1.000 -ATOM 2722 OH2 TIP3 3075 5.503 -17.577 9.355 -0.834 1.520 -ATOM 2723 H1 TIP3 3075 5.815 -17.477 10.265 0.417 1.000 -ATOM 2724 H2 TIP3 3075 5.270 -16.658 9.168 0.417 1.000 -ATOM 2725 OH2 TIP3 3080 -0.560 -7.155 8.739 -0.834 1.520 -ATOM 2726 H1 TIP3 3080 -0.005 -6.889 9.548 0.417 1.000 -ATOM 2727 H2 TIP3 3080 -1.381 -7.120 9.211 0.417 1.000 -ATOM 2728 OH2 TIP3 3085 4.148 -15.531 -18.486 -0.834 1.520 -ATOM 2729 H1 TIP3 3085 3.319 -16.059 -18.560 0.417 1.000 -ATOM 2730 H2 TIP3 3085 4.739 -16.145 -18.019 0.417 1.000 -ATOM 2731 OH2 TIP3 3086 -0.607 -14.241 -18.315 -0.834 1.520 -ATOM 2732 H1 TIP3 3086 -0.998 -15.112 -18.282 0.417 1.000 -ATOM 2733 H2 TIP3 3086 -0.684 -13.918 -17.414 0.417 1.000 -ATOM 2734 OH2 TIP3 3091 -1.363 -11.010 -15.384 -0.834 1.520 -ATOM 2735 H1 TIP3 3091 -2.214 -11.379 -15.487 0.417 1.000 -ATOM 2736 H2 TIP3 3091 -1.406 -10.067 -15.375 0.417 1.000 -ATOM 2737 OH2 TIP3 3092 11.077 -6.160 1.323 -0.834 1.520 -ATOM 2738 H1 TIP3 3092 11.122 -6.696 2.148 0.417 1.000 -ATOM 2739 H2 TIP3 3092 11.515 -5.357 1.528 0.417 1.000 -ATOM 2740 OH2 TIP3 3095 5.917 -11.103 1.213 -0.834 1.520 -ATOM 2741 H1 TIP3 3095 5.210 -11.664 1.602 0.417 1.000 -ATOM 2742 H2 TIP3 3095 5.993 -10.348 1.819 0.417 1.000 -ATOM 2743 OH2 TIP3 3096 -6.669 -15.437 3.416 -0.834 1.520 -ATOM 2744 H1 TIP3 3096 -5.685 -15.228 3.360 0.417 1.000 -ATOM 2745 H2 TIP3 3096 -6.785 -16.322 3.134 0.417 1.000 -ATOM 2746 OH2 TIP3 3097 8.116 -17.558 1.242 -0.834 1.520 -ATOM 2747 H1 TIP3 3097 9.109 -17.560 0.963 0.417 1.000 -ATOM 2748 H2 TIP3 3097 7.651 -17.370 0.470 0.417 1.000 -ATOM 2749 OH2 TIP3 3108 -1.590 -18.027 -11.420 -0.834 1.520 -ATOM 2750 H1 TIP3 3108 -1.342 -17.085 -11.477 0.417 1.000 -ATOM 2751 H2 TIP3 3108 -1.212 -18.433 -12.250 0.417 1.000 -ATOM 2752 OH2 TIP3 3109 2.635 1.010 -17.098 -0.834 1.520 -ATOM 2753 H1 TIP3 3109 2.191 1.343 -17.926 0.417 1.000 -ATOM 2754 H2 TIP3 3109 1.968 0.900 -16.447 0.417 1.000 -ATOM 2755 OH2 TIP3 3111 5.022 -4.978 -13.649 -0.834 1.520 -ATOM 2756 H1 TIP3 3111 5.811 -5.292 -13.043 0.417 1.000 -ATOM 2757 H2 TIP3 3111 4.417 -5.728 -13.596 0.417 1.000 -ATOM 2758 OH2 TIP3 3112 7.034 -6.430 -7.429 -0.834 1.520 -ATOM 2759 H1 TIP3 3112 6.603 -6.608 -6.549 0.417 1.000 -ATOM 2760 H2 TIP3 3112 7.165 -5.453 -7.443 0.417 1.000 -ATOM 2761 OH2 TIP3 3113 2.270 -6.168 2.593 -0.834 1.520 -ATOM 2762 H1 TIP3 3113 1.856 -7.023 2.328 0.417 1.000 -ATOM 2763 H2 TIP3 3113 1.777 -5.963 3.436 0.417 1.000 -ATOM 2764 OH2 TIP3 3114 7.885 -12.638 -6.088 -0.834 1.520 -ATOM 2765 H1 TIP3 3114 7.855 -11.652 -5.985 0.417 1.000 -ATOM 2766 H2 TIP3 3114 7.734 -12.961 -5.175 0.417 1.000 -ATOM 2767 OH2 TIP3 3115 1.606 -9.701 -10.636 -0.834 1.520 -ATOM 2768 H1 TIP3 3115 2.239 -10.380 -10.471 0.417 1.000 -ATOM 2769 H2 TIP3 3115 0.962 -9.778 -9.901 0.417 1.000 -ATOM 2770 OH2 TIP3 3116 9.353 -2.993 -3.791 -0.834 1.520 -ATOM 2771 H1 TIP3 3116 9.824 -2.142 -3.986 0.417 1.000 -ATOM 2772 H2 TIP3 3116 8.983 -2.876 -2.909 0.417 1.000 -ATOM 2773 OH2 TIP3 3119 9.131 -17.734 5.022 -0.834 1.520 -ATOM 2774 H1 TIP3 3119 8.503 -17.550 5.750 0.417 1.000 -ATOM 2775 H2 TIP3 3119 8.526 -18.082 4.334 0.417 1.000 -ATOM 2776 OH2 TIP3 3121 -10.163 -14.878 17.608 -0.834 1.520 -ATOM 2777 H1 TIP3 3121 -10.365 -13.977 17.219 0.417 1.000 -ATOM 2778 H2 TIP3 3121 -9.288 -15.023 17.286 0.417 1.000 -ATOM 2779 OH2 TIP3 3129 4.355 -11.463 -17.874 -0.834 1.520 -ATOM 2780 H1 TIP3 3129 4.816 -11.037 -18.577 0.417 1.000 -ATOM 2781 H2 TIP3 3129 4.086 -12.364 -18.172 0.417 1.000 -ATOM 2782 OH2 TIP3 3131 2.645 -7.781 -16.872 -0.834 1.520 -ATOM 2783 H1 TIP3 3131 1.963 -7.185 -16.501 0.417 1.000 -ATOM 2784 H2 TIP3 3131 2.260 -8.726 -16.685 0.417 1.000 -ATOM 2785 OH2 TIP3 3134 -8.599 1.480 -2.864 -0.834 1.520 -ATOM 2786 H1 TIP3 3134 -8.478 2.112 -2.144 0.417 1.000 -ATOM 2787 H2 TIP3 3134 -8.745 0.672 -2.364 0.417 1.000 -ATOM 2788 OH2 TIP3 3136 -8.713 -18.565 12.202 -0.834 1.520 -ATOM 2789 H1 TIP3 3136 -8.892 -19.166 12.927 0.417 1.000 -ATOM 2790 H2 TIP3 3136 -7.696 -18.588 12.059 0.417 1.000 -ATOM 2791 OH2 TIP3 3138 18.246 -6.323 3.324 -0.834 1.520 -ATOM 2792 H1 TIP3 3138 19.156 -6.048 3.010 0.417 1.000 -ATOM 2793 H2 TIP3 3138 17.828 -6.631 2.503 0.417 1.000 -ATOM 2794 OH2 TIP3 3139 -0.094 -17.234 9.558 -0.834 1.520 -ATOM 2795 H1 TIP3 3139 -0.018 -17.998 10.079 0.417 1.000 -ATOM 2796 H2 TIP3 3139 -0.835 -17.441 8.924 0.417 1.000 -ATOM 2797 OH2 TIP3 3140 4.170 -18.321 1.197 -0.834 1.520 -ATOM 2798 H1 TIP3 3140 4.257 -18.009 2.097 0.417 1.000 -ATOM 2799 H2 TIP3 3140 4.268 -19.244 1.263 0.417 1.000 -ATOM 2800 OH2 TIP3 3142 6.496 -13.441 13.805 -0.834 1.520 -ATOM 2801 H1 TIP3 3142 5.805 -13.694 14.480 0.417 1.000 -ATOM 2802 H2 TIP3 3142 7.375 -13.567 14.338 0.417 1.000 -ATOM 2803 OH2 TIP3 3143 -2.406 -16.456 19.059 -0.834 1.520 -ATOM 2804 H1 TIP3 3143 -3.370 -16.417 18.930 0.417 1.000 -ATOM 2805 H2 TIP3 3143 -2.110 -15.759 18.477 0.417 1.000 -ATOM 2806 OH2 TIP3 3150 -2.209 -7.740 -15.255 -0.834 1.520 -ATOM 2807 H1 TIP3 3150 -2.671 -6.905 -15.387 0.417 1.000 -ATOM 2808 H2 TIP3 3150 -1.996 -7.698 -14.372 0.417 1.000 -ATOM 2809 OH2 TIP3 3152 2.125 -16.538 -9.547 -0.834 1.520 -ATOM 2810 H1 TIP3 3152 1.341 -16.035 -9.153 0.417 1.000 -ATOM 2811 H2 TIP3 3152 2.548 -15.773 -9.979 0.417 1.000 -ATOM 2812 OH2 TIP3 3154 4.178 -8.790 -3.421 -0.834 1.520 -ATOM 2813 H1 TIP3 3154 5.024 -8.602 -2.974 0.417 1.000 -ATOM 2814 H2 TIP3 3154 4.455 -8.963 -4.352 0.417 1.000 -ATOM 2815 OH2 TIP3 3155 5.254 6.275 2.147 -0.834 1.520 -ATOM 2816 H1 TIP3 3155 5.872 6.369 1.396 0.417 1.000 -ATOM 2817 H2 TIP3 3155 5.551 5.476 2.594 0.417 1.000 -ATOM 2818 OH2 TIP3 3156 8.890 2.243 7.795 -0.834 1.520 -ATOM 2819 H1 TIP3 3156 9.114 2.850 7.064 0.417 1.000 -ATOM 2820 H2 TIP3 3156 9.516 1.585 7.824 0.417 1.000 -ATOM 2821 OH2 TIP3 3157 2.023 7.254 -6.788 -0.834 1.520 -ATOM 2822 H1 TIP3 3157 2.008 7.284 -7.723 0.417 1.000 -ATOM 2823 H2 TIP3 3157 2.707 7.959 -6.570 0.417 1.000 -ATOM 2824 OH2 TIP3 3158 -2.116 -3.737 9.508 -0.834 1.520 -ATOM 2825 H1 TIP3 3158 -2.851 -3.277 9.070 0.417 1.000 -ATOM 2826 H2 TIP3 3158 -1.683 -4.042 8.695 0.417 1.000 -ATOM 2827 OH2 TIP3 3160 8.983 -4.584 15.557 -0.834 1.520 -ATOM 2828 H1 TIP3 3160 9.732 -4.083 15.260 0.417 1.000 -ATOM 2829 H2 TIP3 3160 8.272 -3.946 15.423 0.417 1.000 -ATOM 2830 OH2 TIP3 3161 -0.931 -11.228 8.945 -0.834 1.520 -ATOM 2831 H1 TIP3 3161 -0.314 -11.632 9.594 0.417 1.000 -ATOM 2832 H2 TIP3 3161 -1.601 -11.866 8.762 0.417 1.000 -ATOM 2833 OH2 TIP3 3173 -7.924 -15.355 -6.434 -0.834 1.520 -ATOM 2834 H1 TIP3 3173 -8.000 -14.478 -6.852 0.417 1.000 -ATOM 2835 H2 TIP3 3173 -8.173 -15.165 -5.557 0.417 1.000 -ATOM 2836 OH2 TIP3 3174 9.320 -5.897 -3.171 -0.834 1.520 -ATOM 2837 H1 TIP3 3174 9.585 -5.816 -2.215 0.417 1.000 -ATOM 2838 H2 TIP3 3174 9.594 -5.005 -3.568 0.417 1.000 -ATOM 2839 OH2 TIP3 3175 10.419 -2.121 16.837 -0.834 1.520 -ATOM 2840 H1 TIP3 3175 10.268 -2.517 17.712 0.417 1.000 -ATOM 2841 H2 TIP3 3175 9.631 -1.552 16.760 0.417 1.000 -ATOM 2842 OH2 TIP3 3176 16.521 -1.853 -18.889 -0.834 1.520 -ATOM 2843 H1 TIP3 3176 16.496 -0.915 -18.899 0.417 1.000 -ATOM 2844 H2 TIP3 3176 15.815 -2.083 -18.333 0.417 1.000 -ATOM 2845 OH2 TIP3 3177 -10.869 -18.925 -4.159 -0.834 1.520 -ATOM 2846 H1 TIP3 3177 -10.052 -18.537 -4.588 0.417 1.000 -ATOM 2847 H2 TIP3 3177 -10.887 -19.839 -4.536 0.417 1.000 -ATOM 2848 OH2 TIP3 3178 9.503 7.905 4.858 -0.834 1.520 -ATOM 2849 H1 TIP3 3178 9.654 8.084 5.814 0.417 1.000 -ATOM 2850 H2 TIP3 3178 8.893 7.106 4.845 0.417 1.000 -ATOM 2851 OH2 TIP3 3179 -1.752 4.644 0.171 -0.834 1.520 -ATOM 2852 H1 TIP3 3179 -2.174 5.152 -0.539 0.417 1.000 -ATOM 2853 H2 TIP3 3179 -1.836 3.727 -0.164 0.417 1.000 -ATOM 2854 OH2 TIP3 3180 4.663 -1.034 -0.928 -0.834 1.520 -ATOM 2855 H1 TIP3 3180 4.078 -0.751 -1.619 0.417 1.000 -ATOM 2856 H2 TIP3 3180 4.908 -1.975 -1.190 0.417 1.000 -ATOM 2857 OH2 TIP3 3181 -3.661 -8.738 5.272 -0.834 1.520 -ATOM 2858 H1 TIP3 3181 -3.701 -9.359 4.513 0.417 1.000 -ATOM 2859 H2 TIP3 3181 -3.936 -9.341 5.958 0.417 1.000 -ATOM 2860 OH2 TIP3 3182 10.866 4.333 11.715 -0.834 1.520 -ATOM 2861 H1 TIP3 3182 10.291 4.860 11.204 0.417 1.000 -ATOM 2862 H2 TIP3 3182 10.943 3.530 11.132 0.417 1.000 -ATOM 2863 OH2 TIP3 3183 2.338 -11.596 12.676 -0.834 1.520 -ATOM 2864 H1 TIP3 3183 2.496 -12.455 13.182 0.417 1.000 -ATOM 2865 H2 TIP3 3183 3.035 -11.030 12.963 0.417 1.000 -ATOM 2866 OH2 TIP3 3184 -14.095 -4.067 12.791 -0.834 1.520 -ATOM 2867 H1 TIP3 3184 -14.027 -3.328 13.406 0.417 1.000 -ATOM 2868 H2 TIP3 3184 -14.321 -4.839 13.336 0.417 1.000 -ATOM 2869 OH2 TIP3 3187 11.175 -13.295 -7.411 -0.834 1.520 -ATOM 2870 H1 TIP3 3187 11.233 -12.872 -8.324 0.417 1.000 -ATOM 2871 H2 TIP3 3187 10.501 -14.026 -7.611 0.417 1.000 -ATOM 2872 OH2 TIP3 3189 -3.439 -5.988 14.908 -0.834 1.520 -ATOM 2873 H1 TIP3 3189 -4.198 -5.655 15.417 0.417 1.000 -ATOM 2874 H2 TIP3 3189 -3.485 -6.910 14.965 0.417 1.000 -ATOM 2875 OH2 TIP3 3192 3.294 -3.286 -11.647 -0.834 1.520 -ATOM 2876 H1 TIP3 3192 3.759 -2.525 -11.329 0.417 1.000 -ATOM 2877 H2 TIP3 3192 4.011 -3.670 -12.048 0.417 1.000 -ATOM 2878 OH2 TIP3 3193 19.192 7.065 -5.877 -0.834 1.520 -ATOM 2879 H1 TIP3 3193 19.122 6.450 -6.610 0.417 1.000 -ATOM 2880 H2 TIP3 3193 19.538 6.461 -5.188 0.417 1.000 -ATOM 2881 OH2 TIP3 3195 2.513 4.059 -1.016 -0.834 1.520 -ATOM 2882 H1 TIP3 3195 3.107 4.881 -1.044 0.417 1.000 -ATOM 2883 H2 TIP3 3195 1.878 4.220 -0.286 0.417 1.000 -ATOM 2884 OH2 TIP3 3196 -12.306 -11.662 -3.313 -0.834 1.520 -ATOM 2885 H1 TIP3 3196 -12.677 -11.919 -2.492 0.417 1.000 -ATOM 2886 H2 TIP3 3196 -13.009 -11.524 -3.928 0.417 1.000 -ATOM 2887 OH2 TIP3 3197 -4.443 12.531 -2.210 -0.834 1.520 -ATOM 2888 H1 TIP3 3197 -3.542 12.883 -2.545 0.417 1.000 -ATOM 2889 H2 TIP3 3197 -4.760 11.941 -2.968 0.417 1.000 -ATOM 2890 OH2 TIP3 3198 -2.945 -7.061 -0.247 -0.834 1.520 -ATOM 2891 H1 TIP3 3198 -2.961 -8.039 -0.450 0.417 1.000 -ATOM 2892 H2 TIP3 3198 -1.990 -6.871 -0.456 0.417 1.000 -ATOM 2893 OH2 TIP3 3199 11.147 15.582 7.692 -0.834 1.520 -ATOM 2894 H1 TIP3 3199 11.959 16.047 7.697 0.417 1.000 -ATOM 2895 H2 TIP3 3199 11.357 14.665 7.740 0.417 1.000 -ATOM 2896 OH2 TIP3 3200 15.853 4.245 -5.160 -0.834 1.520 -ATOM 2897 H1 TIP3 3200 15.833 3.388 -4.720 0.417 1.000 -ATOM 2898 H2 TIP3 3200 15.889 3.988 -6.115 0.417 1.000 -ATOM 2899 OH2 TIP3 3201 5.032 3.820 12.217 -0.834 1.520 -ATOM 2900 H1 TIP3 3201 4.738 4.411 11.470 0.417 1.000 -ATOM 2901 H2 TIP3 3201 5.931 4.128 12.387 0.417 1.000 -ATOM 2902 OH2 TIP3 3202 5.872 2.902 -1.402 -0.834 1.520 -ATOM 2903 H1 TIP3 3202 6.770 2.404 -1.282 0.417 1.000 -ATOM 2904 H2 TIP3 3202 5.176 2.248 -1.501 0.417 1.000 -ATOM 2905 OH2 TIP3 3203 11.371 0.493 12.276 -0.834 1.520 -ATOM 2906 H1 TIP3 3203 10.719 0.309 11.600 0.417 1.000 -ATOM 2907 H2 TIP3 3203 12.011 1.039 11.867 0.417 1.000 -ATOM 2908 OH2 TIP3 3204 -0.008 9.459 13.269 -0.834 1.520 -ATOM 2909 H1 TIP3 3204 -0.942 9.216 13.497 0.417 1.000 -ATOM 2910 H2 TIP3 3204 0.183 10.155 13.886 0.417 1.000 -ATOM 2911 OH2 TIP3 3205 7.205 -2.704 14.051 -0.834 1.520 -ATOM 2912 H1 TIP3 3205 6.828 -1.978 14.599 0.417 1.000 -ATOM 2913 H2 TIP3 3205 6.431 -2.967 13.469 0.417 1.000 -ATOM 2914 OH2 TIP3 3210 1.944 -9.429 19.387 -0.834 1.520 -ATOM 2915 H1 TIP3 3210 1.370 -9.907 19.974 0.417 1.000 -ATOM 2916 H2 TIP3 3210 2.066 -10.048 18.625 0.417 1.000 -ATOM 2917 OH2 TIP3 3214 4.647 -7.840 -13.525 -0.834 1.520 -ATOM 2918 H1 TIP3 3214 4.753 -8.240 -14.361 0.417 1.000 -ATOM 2919 H2 TIP3 3214 5.517 -7.507 -13.337 0.417 1.000 -ATOM 2920 OH2 TIP3 3215 -0.013 -0.926 -5.059 -0.834 1.520 -ATOM 2921 H1 TIP3 3215 -0.831 -0.652 -4.691 0.417 1.000 -ATOM 2922 H2 TIP3 3215 0.655 -0.477 -4.533 0.417 1.000 -ATOM 2923 OH2 TIP3 3216 4.248 8.156 4.606 -0.834 1.520 -ATOM 2924 H1 TIP3 3216 3.326 8.246 4.557 0.417 1.000 -ATOM 2925 H2 TIP3 3216 4.522 8.264 3.722 0.417 1.000 -ATOM 2926 OH2 TIP3 3218 -8.088 -6.788 -0.274 -0.834 1.520 -ATOM 2927 H1 TIP3 3218 -7.146 -6.775 -0.119 0.417 1.000 -ATOM 2928 H2 TIP3 3218 -8.496 -7.236 0.555 0.417 1.000 -ATOM 2929 OH2 TIP3 3219 1.300 -2.592 3.235 -0.834 1.520 -ATOM 2930 H1 TIP3 3219 0.535 -2.924 2.776 0.417 1.000 -ATOM 2931 H2 TIP3 3219 1.183 -3.037 4.104 0.417 1.000 -ATOM 2932 OH2 TIP3 3220 2.705 14.717 -10.818 -0.834 1.520 -ATOM 2933 H1 TIP3 3220 3.278 15.331 -11.298 0.417 1.000 -ATOM 2934 H2 TIP3 3220 3.304 14.311 -10.142 0.417 1.000 -ATOM 2935 OH2 TIP3 3221 -1.733 1.738 19.016 -0.834 1.520 -ATOM 2936 H1 TIP3 3221 -1.397 2.610 18.851 0.417 1.000 -ATOM 2937 H2 TIP3 3221 -0.911 1.151 18.956 0.417 1.000 -ATOM 2938 OH2 TIP3 3222 6.607 3.205 9.543 -0.834 1.520 -ATOM 2939 H1 TIP3 3222 6.786 4.139 9.271 0.417 1.000 -ATOM 2940 H2 TIP3 3222 7.353 2.697 9.149 0.417 1.000 -ATOM 2941 OH2 TIP3 3223 -14.885 -0.180 2.632 -0.834 1.520 -ATOM 2942 H1 TIP3 3223 -15.730 -0.248 2.092 0.417 1.000 -ATOM 2943 H2 TIP3 3223 -14.200 -0.415 1.996 0.417 1.000 -ATOM 2944 OH2 TIP3 3224 3.721 -1.117 13.019 -0.834 1.520 -ATOM 2945 H1 TIP3 3224 2.924 -0.628 13.254 0.417 1.000 -ATOM 2946 H2 TIP3 3224 4.318 -0.492 12.617 0.417 1.000 -ATOM 2947 OH2 TIP3 3225 17.593 -3.159 17.197 -0.834 1.520 -ATOM 2948 H1 TIP3 3225 16.996 -2.815 17.832 0.417 1.000 -ATOM 2949 H2 TIP3 3225 18.385 -2.745 17.381 0.417 1.000 -ATOM 2950 OH2 TIP3 3226 -7.307 12.804 14.562 -0.834 1.520 -ATOM 2951 H1 TIP3 3226 -7.284 12.558 13.617 0.417 1.000 -ATOM 2952 H2 TIP3 3226 -6.855 12.069 15.024 0.417 1.000 -ATOM 2953 OH2 TIP3 3227 -6.440 -2.066 8.175 -0.834 1.520 -ATOM 2954 H1 TIP3 3227 -6.613 -1.137 8.340 0.417 1.000 -ATOM 2955 H2 TIP3 3227 -5.578 -2.096 8.682 0.417 1.000 -ATOM 2956 OH2 TIP3 3229 6.458 4.748 15.379 -0.834 1.520 -ATOM 2957 H1 TIP3 3229 7.244 4.814 15.919 0.417 1.000 -ATOM 2958 H2 TIP3 3229 5.726 5.082 15.994 0.417 1.000 -ATOM 2959 OH2 TIP3 3233 8.583 10.824 -16.317 -0.834 1.520 -ATOM 2960 H1 TIP3 3233 8.557 10.908 -15.305 0.417 1.000 -ATOM 2961 H2 TIP3 3233 9.160 10.067 -16.388 0.417 1.000 -ATOM 2962 OH2 TIP3 3234 -3.682 2.811 -11.445 -0.834 1.520 -ATOM 2963 H1 TIP3 3234 -3.968 3.275 -10.656 0.417 1.000 -ATOM 2964 H2 TIP3 3234 -2.714 2.916 -11.418 0.417 1.000 -ATOM 2965 OH2 TIP3 3236 -7.386 7.273 -6.304 -0.834 1.520 -ATOM 2966 H1 TIP3 3236 -6.876 7.950 -6.693 0.417 1.000 -ATOM 2967 H2 TIP3 3236 -7.932 7.765 -5.707 0.417 1.000 -ATOM 2968 OH2 TIP3 3237 1.939 1.100 -4.672 -0.834 1.520 -ATOM 2969 H1 TIP3 3237 2.635 0.640 -5.179 0.417 1.000 -ATOM 2970 H2 TIP3 3237 2.375 1.374 -3.820 0.417 1.000 -ATOM 2971 OH2 TIP3 3238 7.007 5.179 -0.237 -0.834 1.520 -ATOM 2972 H1 TIP3 3238 6.673 4.409 -0.768 0.417 1.000 -ATOM 2973 H2 TIP3 3238 7.930 5.173 -0.543 0.417 1.000 -ATOM 2974 OH2 TIP3 3239 8.419 15.932 -3.784 -0.834 1.520 -ATOM 2975 H1 TIP3 3239 9.084 16.176 -4.443 0.417 1.000 -ATOM 2976 H2 TIP3 3239 7.622 16.176 -4.151 0.417 1.000 -ATOM 2977 OH2 TIP3 3240 -4.214 8.405 -0.705 -0.834 1.520 -ATOM 2978 H1 TIP3 3240 -4.857 9.196 -0.625 0.417 1.000 -ATOM 2979 H2 TIP3 3240 -4.275 8.200 -1.660 0.417 1.000 -ATOM 2980 OH2 TIP3 3241 0.882 11.566 -12.500 -0.834 1.520 -ATOM 2981 H1 TIP3 3241 1.771 11.259 -12.636 0.417 1.000 -ATOM 2982 H2 TIP3 3241 1.048 12.516 -12.610 0.417 1.000 -ATOM 2983 OH2 TIP3 3242 -1.210 -0.672 -0.896 -0.834 1.520 -ATOM 2984 H1 TIP3 3242 -1.736 -0.728 -0.114 0.417 1.000 -ATOM 2985 H2 TIP3 3242 -1.414 -1.472 -1.402 0.417 1.000 -ATOM 2986 OH2 TIP3 3243 9.487 11.469 -5.683 -0.834 1.520 -ATOM 2987 H1 TIP3 3243 8.967 12.221 -5.467 0.417 1.000 -ATOM 2988 H2 TIP3 3243 9.536 10.898 -4.878 0.417 1.000 -ATOM 2989 OH2 TIP3 3244 -0.713 2.590 -11.270 -0.834 1.520 -ATOM 2990 H1 TIP3 3244 -0.232 2.291 -11.966 0.417 1.000 -ATOM 2991 H2 TIP3 3244 -0.677 1.826 -10.633 0.417 1.000 -ATOM 2992 OH2 TIP3 3245 -10.000 -5.207 9.822 -0.834 1.520 -ATOM 2993 H1 TIP3 3245 -10.878 -5.274 9.406 0.417 1.000 -ATOM 2994 H2 TIP3 3245 -10.180 -4.696 10.640 0.417 1.000 -ATOM 2995 OH2 TIP3 3246 -0.780 -4.326 15.479 -0.834 1.520 -ATOM 2996 H1 TIP3 3246 -0.321 -4.929 14.897 0.417 1.000 -ATOM 2997 H2 TIP3 3246 -1.641 -4.608 15.439 0.417 1.000 -ATOM 2998 OH2 TIP3 3247 -4.823 7.294 6.773 -0.834 1.520 -ATOM 2999 H1 TIP3 3247 -4.781 6.636 7.565 0.417 1.000 -ATOM 3000 H2 TIP3 3247 -3.965 7.115 6.274 0.417 1.000 -ATOM 3001 OH2 TIP3 3257 6.173 6.160 -17.953 -0.834 1.520 -ATOM 3002 H1 TIP3 3257 5.814 7.030 -18.388 0.417 1.000 -ATOM 3003 H2 TIP3 3257 7.180 6.313 -18.001 0.417 1.000 -ATOM 3004 OH2 TIP3 3258 4.149 -1.112 -13.467 -0.834 1.520 -ATOM 3005 H1 TIP3 3258 4.021 -0.834 -14.399 0.417 1.000 -ATOM 3006 H2 TIP3 3258 5.032 -0.767 -13.278 0.417 1.000 -ATOM 3007 OH2 TIP3 3261 2.392 6.765 7.367 -0.834 1.520 -ATOM 3008 H1 TIP3 3261 1.649 7.484 7.268 0.417 1.000 -ATOM 3009 H2 TIP3 3261 2.874 7.108 8.071 0.417 1.000 -ATOM 3010 OH2 TIP3 3262 -4.796 3.340 -8.871 -0.834 1.520 -ATOM 3011 H1 TIP3 3262 -5.433 3.666 -8.159 0.417 1.000 -ATOM 3012 H2 TIP3 3262 -3.964 3.766 -8.618 0.417 1.000 -ATOM 3013 OH2 TIP3 3263 2.974 -8.613 5.453 -0.834 1.520 -ATOM 3014 H1 TIP3 3263 2.750 -7.707 5.160 0.417 1.000 -ATOM 3015 H2 TIP3 3263 2.131 -8.896 5.792 0.417 1.000 -ATOM 3016 OH2 TIP3 3264 -3.114 16.905 7.161 -0.834 1.520 -ATOM 3017 H1 TIP3 3264 -3.594 16.962 6.323 0.417 1.000 -ATOM 3018 H2 TIP3 3264 -3.702 17.368 7.757 0.417 1.000 -ATOM 3019 OH2 TIP3 3265 3.208 14.407 5.395 -0.834 1.520 -ATOM 3020 H1 TIP3 3265 4.083 13.970 5.343 0.417 1.000 -ATOM 3021 H2 TIP3 3265 2.947 14.231 6.346 0.417 1.000 -ATOM 3022 OH2 TIP3 3266 12.701 2.303 18.307 -0.834 1.520 -ATOM 3023 H1 TIP3 3266 12.114 1.754 18.844 0.417 1.000 -ATOM 3024 H2 TIP3 3266 13.510 1.724 18.221 0.417 1.000 -ATOM 3025 OH2 TIP3 3274 6.353 1.381 -15.560 -0.834 1.520 -ATOM 3026 H1 TIP3 3274 5.914 0.546 -15.729 0.417 1.000 -ATOM 3027 H2 TIP3 3274 5.689 1.985 -15.148 0.417 1.000 -ATOM 3028 OH2 TIP3 3280 5.090 2.311 -9.209 -0.834 1.520 -ATOM 3029 H1 TIP3 3280 5.958 2.594 -9.665 0.417 1.000 -ATOM 3030 H2 TIP3 3280 5.437 1.952 -8.332 0.417 1.000 -ATOM 3031 OH2 TIP3 3282 9.435 8.189 -8.513 -0.834 1.520 -ATOM 3032 H1 TIP3 3282 9.147 9.086 -8.564 0.417 1.000 -ATOM 3033 H2 TIP3 3282 9.190 7.885 -7.613 0.417 1.000 -ATOM 3034 OH2 TIP3 3283 3.782 19.218 -6.907 -0.834 1.520 -ATOM 3035 H1 TIP3 3283 3.139 19.036 -7.637 0.417 1.000 -ATOM 3036 H2 TIP3 3283 4.011 18.329 -6.606 0.417 1.000 -ATOM 3037 OH2 TIP3 3284 5.829 15.646 -1.832 -0.834 1.520 -ATOM 3038 H1 TIP3 3284 6.530 15.401 -1.148 0.417 1.000 -ATOM 3039 H2 TIP3 3284 5.718 16.638 -1.656 0.417 1.000 -ATOM 3040 OH2 TIP3 3285 15.307 15.122 2.062 -0.834 1.520 -ATOM 3041 H1 TIP3 3285 14.845 14.314 1.710 0.417 1.000 -ATOM 3042 H2 TIP3 3285 14.612 15.547 2.606 0.417 1.000 -ATOM 3043 OH2 TIP3 3286 15.633 2.799 8.763 -0.834 1.520 -ATOM 3044 H1 TIP3 3286 15.279 3.676 8.900 0.417 1.000 -ATOM 3045 H2 TIP3 3286 15.497 2.405 9.622 0.417 1.000 -ATOM 3046 OH2 TIP3 3287 -17.255 8.967 9.795 -0.834 1.520 -ATOM 3047 H1 TIP3 3287 -18.187 8.948 10.040 0.417 1.000 -ATOM 3048 H2 TIP3 3287 -16.968 9.865 10.120 0.417 1.000 -ATOM 3049 OH2 TIP3 3291 5.296 2.140 18.214 -0.834 1.520 -ATOM 3050 H1 TIP3 3291 5.098 2.097 17.242 0.417 1.000 -ATOM 3051 H2 TIP3 3291 6.213 2.144 18.202 0.417 1.000 -ATOM 3052 OH2 TIP3 3296 12.370 -0.997 -18.176 -0.834 1.520 -ATOM 3053 H1 TIP3 3296 12.660 -1.227 -19.123 0.417 1.000 -ATOM 3054 H2 TIP3 3296 11.974 -0.091 -18.275 0.417 1.000 -ATOM 3055 OH2 TIP3 3298 7.743 17.380 -14.522 -0.834 1.520 -ATOM 3056 H1 TIP3 3298 8.278 18.222 -14.380 0.417 1.000 -ATOM 3057 H2 TIP3 3298 7.645 17.266 -15.494 0.417 1.000 -ATOM 3058 OH2 TIP3 3301 12.733 10.590 -10.624 -0.834 1.520 -ATOM 3059 H1 TIP3 3301 12.538 10.142 -11.436 0.417 1.000 -ATOM 3060 H2 TIP3 3301 13.111 9.846 -10.097 0.417 1.000 -ATOM 3061 OH2 TIP3 3302 4.767 7.580 -15.480 -0.834 1.520 -ATOM 3062 H1 TIP3 3302 4.390 6.725 -15.689 0.417 1.000 -ATOM 3063 H2 TIP3 3302 4.787 7.986 -16.355 0.417 1.000 -ATOM 3064 OH2 TIP3 3303 -5.830 3.047 0.478 -0.834 1.520 -ATOM 3065 H1 TIP3 3303 -4.985 2.883 0.902 0.417 1.000 -ATOM 3066 H2 TIP3 3303 -6.243 2.177 0.579 0.417 1.000 -ATOM 3067 OH2 TIP3 3304 -6.837 0.489 0.636 -0.834 1.520 -ATOM 3068 H1 TIP3 3304 -7.550 0.171 0.104 0.417 1.000 -ATOM 3069 H2 TIP3 3304 -6.780 -0.125 1.393 0.417 1.000 -ATOM 3070 OH2 TIP3 3305 -3.720 1.031 7.412 -0.834 1.520 -ATOM 3071 H1 TIP3 3305 -4.659 1.271 7.560 0.417 1.000 -ATOM 3072 H2 TIP3 3305 -3.720 1.124 6.419 0.417 1.000 -ATOM 3073 OH2 TIP3 3306 6.330 6.152 9.299 -0.834 1.520 -ATOM 3074 H1 TIP3 3306 5.913 6.992 9.061 0.417 1.000 -ATOM 3075 H2 TIP3 3306 5.617 5.760 9.754 0.417 1.000 -ATOM 3076 OH2 TIP3 3307 8.097 9.556 12.589 -0.834 1.520 -ATOM 3077 H1 TIP3 3307 8.754 9.669 11.895 0.417 1.000 -ATOM 3078 H2 TIP3 3307 8.513 9.950 13.310 0.417 1.000 -ATOM 3079 OH2 TIP3 3308 9.510 18.708 -0.676 -0.834 1.520 -ATOM 3080 H1 TIP3 3308 10.347 19.166 -0.983 0.417 1.000 -ATOM 3081 H2 TIP3 3308 9.207 18.357 -1.520 0.417 1.000 -ATOM 3082 OH2 TIP3 3310 -6.944 11.407 18.941 -0.834 1.520 -ATOM 3083 H1 TIP3 3310 -7.190 11.307 19.873 0.417 1.000 -ATOM 3084 H2 TIP3 3310 -6.290 10.737 18.787 0.417 1.000 -ATOM 3085 OH2 TIP3 3317 12.713 6.595 -17.591 -0.834 1.520 -ATOM 3086 H1 TIP3 3317 13.443 6.598 -18.204 0.417 1.000 -ATOM 3087 H2 TIP3 3317 12.159 5.868 -17.831 0.417 1.000 -ATOM 3088 OH2 TIP3 3319 2.940 14.912 -17.316 -0.834 1.520 -ATOM 3089 H1 TIP3 3319 3.335 14.036 -17.466 0.417 1.000 -ATOM 3090 H2 TIP3 3319 2.084 14.834 -17.682 0.417 1.000 -ATOM 3091 OH2 TIP3 3321 2.821 14.248 -5.771 -0.834 1.520 -ATOM 3092 H1 TIP3 3321 2.847 13.705 -4.945 0.417 1.000 -ATOM 3093 H2 TIP3 3321 2.283 15.020 -5.359 0.417 1.000 -ATOM 3094 OH2 TIP3 3322 0.476 15.705 -15.355 -0.834 1.520 -ATOM 3095 H1 TIP3 3322 -0.320 15.570 -15.913 0.417 1.000 -ATOM 3096 H2 TIP3 3322 0.828 16.523 -15.698 0.417 1.000 -ATOM 3097 OH2 TIP3 3325 14.105 16.903 7.844 -0.834 1.520 -ATOM 3098 H1 TIP3 3325 14.244 17.892 7.722 0.417 1.000 -ATOM 3099 H2 TIP3 3325 14.912 16.594 7.383 0.417 1.000 -ATOM 3100 OH2 TIP3 3326 1.948 15.554 -2.109 -0.834 1.520 -ATOM 3101 H1 TIP3 3326 2.717 15.377 -1.590 0.417 1.000 -ATOM 3102 H2 TIP3 3326 1.530 16.190 -1.504 0.417 1.000 -ATOM 3103 OH2 TIP3 3327 -8.740 17.650 9.834 -0.834 1.520 -ATOM 3104 H1 TIP3 3327 -8.171 18.387 9.997 0.417 1.000 -ATOM 3105 H2 TIP3 3327 -8.734 17.189 10.668 0.417 1.000 -ATOM 3106 OH2 TIP3 3331 2.641 3.390 8.573 -0.834 1.520 -ATOM 3107 H1 TIP3 3331 1.932 3.613 7.913 0.417 1.000 -ATOM 3108 H2 TIP3 3331 3.462 3.561 8.011 0.417 1.000 -ATOM 3109 OH2 TIP3 3341 0.039 14.865 -11.770 -0.834 1.520 -ATOM 3110 H1 TIP3 3341 0.969 14.652 -11.619 0.417 1.000 -ATOM 3111 H2 TIP3 3341 -0.410 14.352 -11.049 0.417 1.000 -ATOM 3112 OH2 TIP3 3342 -6.120 12.413 -10.947 -0.834 1.520 -ATOM 3113 H1 TIP3 3342 -6.887 12.809 -10.546 0.417 1.000 -ATOM 3114 H2 TIP3 3342 -6.460 11.667 -11.547 0.417 1.000 -ATOM 3115 OH2 TIP3 3344 3.196 18.709 -11.576 -0.834 1.520 -ATOM 3116 H1 TIP3 3344 3.539 17.839 -11.882 0.417 1.000 -ATOM 3117 H2 TIP3 3344 2.601 19.066 -12.325 0.417 1.000 -ATOM 3118 OH2 TIP3 3346 11.502 13.556 9.930 -0.834 1.520 -ATOM 3119 H1 TIP3 3346 10.719 13.799 10.505 0.417 1.000 -ATOM 3120 H2 TIP3 3346 12.094 14.329 9.973 0.417 1.000 -ATOM 3121 OH2 TIP3 3347 3.824 8.288 1.326 -0.834 1.520 -ATOM 3122 H1 TIP3 3347 3.081 7.755 1.061 0.417 1.000 -ATOM 3123 H2 TIP3 3347 4.386 7.611 1.752 0.417 1.000 -ATOM 3124 OH2 TIP3 3348 11.386 9.854 0.289 -0.834 1.520 -ATOM 3125 H1 TIP3 3348 11.241 9.291 1.089 0.417 1.000 -ATOM 3126 H2 TIP3 3348 10.965 10.697 0.583 0.417 1.000 -ATOM 3127 OH2 TIP3 3349 -4.743 15.885 1.557 -0.834 1.520 -ATOM 3128 H1 TIP3 3349 -5.635 16.151 1.787 0.417 1.000 -ATOM 3129 H2 TIP3 3349 -4.867 15.206 0.896 0.417 1.000 -ATOM 3130 OH2 TIP3 3354 8.719 3.045 14.191 -0.834 1.520 -ATOM 3131 H1 TIP3 3354 9.021 3.376 15.032 0.417 1.000 -ATOM 3132 H2 TIP3 3354 8.093 3.809 13.941 0.417 1.000 -ATOM 3133 OH2 TIP3 3355 7.584 18.273 13.328 -0.834 1.520 -ATOM 3134 H1 TIP3 3355 8.182 17.595 13.635 0.417 1.000 -ATOM 3135 H2 TIP3 3355 7.542 18.939 14.090 0.417 1.000 -ATOM 3136 OH2 TIP3 3360 -6.296 14.221 -18.664 -0.834 1.520 -ATOM 3137 H1 TIP3 3360 -7.201 13.899 -18.594 0.417 1.000 -ATOM 3138 H2 TIP3 3360 -6.435 15.045 -19.002 0.417 1.000 -ATOM 3139 OH2 TIP3 3361 16.938 12.388 -12.789 -0.834 1.520 -ATOM 3140 H1 TIP3 3361 16.351 12.372 -13.508 0.417 1.000 -ATOM 3141 H2 TIP3 3361 16.380 11.959 -12.054 0.417 1.000 -ATOM 3142 OH2 TIP3 3362 15.592 19.396 -5.750 -0.834 1.520 -ATOM 3143 H1 TIP3 3362 15.721 18.731 -5.066 0.417 1.000 -ATOM 3144 H2 TIP3 3362 14.608 19.476 -5.822 0.417 1.000 -ATOM 3145 OH2 TIP3 3364 4.484 16.412 -7.049 -0.834 1.520 -ATOM 3146 H1 TIP3 3364 4.485 16.471 -8.026 0.417 1.000 -ATOM 3147 H2 TIP3 3364 4.158 15.531 -6.932 0.417 1.000 -ATOM 3148 OH2 TIP3 3365 -5.278 11.521 3.300 -0.834 1.520 -ATOM 3149 H1 TIP3 3365 -5.073 11.386 2.378 0.417 1.000 -ATOM 3150 H2 TIP3 3365 -5.063 12.455 3.478 0.417 1.000 -ATOM 3151 OH2 TIP3 3367 1.063 7.441 0.605 -0.834 1.520 -ATOM 3152 H1 TIP3 3367 0.963 7.681 -0.336 0.417 1.000 -ATOM 3153 H2 TIP3 3367 0.342 7.928 0.966 0.417 1.000 -ATOM 3154 OH2 TIP3 3369 11.035 9.187 18.107 -0.834 1.520 -ATOM 3155 H1 TIP3 3369 11.918 8.994 17.751 0.417 1.000 -ATOM 3156 H2 TIP3 3369 10.832 9.960 17.584 0.417 1.000 -ATOM 3157 OH2 TIP3 3381 -3.343 18.461 -13.027 -0.834 1.520 -ATOM 3158 H1 TIP3 3381 -2.652 17.883 -12.653 0.417 1.000 -ATOM 3159 H2 TIP3 3381 -2.867 19.239 -13.300 0.417 1.000 -ATOM 3160 OH2 TIP3 3382 -3.963 10.640 -18.572 -0.834 1.520 -ATOM 3161 H1 TIP3 3382 -3.743 9.742 -18.476 0.417 1.000 -ATOM 3162 H2 TIP3 3382 -4.109 10.862 -17.613 0.417 1.000 -ATOM 3163 OH2 TIP3 3386 -7.440 16.914 -1.482 -0.834 1.520 -ATOM 3164 H1 TIP3 3386 -7.689 16.841 -0.540 0.417 1.000 -ATOM 3165 H2 TIP3 3386 -6.828 16.216 -1.656 0.417 1.000 -ATOM 3166 OH2 TIP3 3390 3.102 17.270 9.478 -0.834 1.520 -ATOM 3167 H1 TIP3 3390 2.345 16.778 9.860 0.417 1.000 -ATOM 3168 H2 TIP3 3390 3.430 17.711 10.304 0.417 1.000 -ATOM 3169 OH2 TIP3 3402 9.449 12.516 -18.554 -0.834 1.520 -ATOM 3170 H1 TIP3 3402 8.981 11.946 -17.918 0.417 1.000 -ATOM 3171 H2 TIP3 3402 9.079 12.287 -19.363 0.417 1.000 -ATOM 3172 OH2 TIP3 3407 12.538 16.315 -13.204 -0.834 1.520 -ATOM 3173 H1 TIP3 3407 11.589 16.133 -13.445 0.417 1.000 -ATOM 3174 H2 TIP3 3407 12.745 16.978 -13.943 0.417 1.000 -ATOM 3175 OH2 TIP3 3408 5.857 9.295 -8.119 -0.834 1.520 -ATOM 3176 H1 TIP3 3408 5.568 9.647 -8.984 0.417 1.000 -ATOM 3177 H2 TIP3 3408 6.103 8.399 -8.327 0.417 1.000 -ATOM 3178 OH2 TIP3 3426 -4.258 -18.175 -10.226 -0.834 1.520 -ATOM 3179 H1 TIP3 3426 -3.301 -18.446 -10.271 0.417 1.000 -ATOM 3180 H2 TIP3 3426 -4.751 -18.826 -9.629 0.417 1.000 -ATOM 3181 OH2 TIP3 3429 12.174 18.309 -15.121 -0.834 1.520 -ATOM 3182 H1 TIP3 3429 11.952 19.107 -15.632 0.417 1.000 -ATOM 3183 H2 TIP3 3429 12.026 18.641 -14.203 0.417 1.000 -ATOM 3184 OH2 TIP3 3434 3.636 -17.881 13.250 -0.834 1.520 -ATOM 3185 H1 TIP3 3434 3.720 -18.455 14.027 0.417 1.000 -ATOM 3186 H2 TIP3 3434 2.745 -17.597 13.252 0.417 1.000 -ATOM 3187 OH2 TIP3 3442 -3.430 -6.877 -18.044 -0.834 1.520 -ATOM 3188 H1 TIP3 3442 -2.520 -6.979 -18.452 0.417 1.000 -ATOM 3189 H2 TIP3 3442 -3.718 -5.904 -18.338 0.417 1.000 -ATOM 3190 OH2 TIP3 3447 -8.684 -17.727 -12.044 -0.834 1.520 -ATOM 3191 H1 TIP3 3447 -8.609 -18.642 -11.849 0.417 1.000 -ATOM 3192 H2 TIP3 3447 -9.577 -17.521 -11.819 0.417 1.000 -ATOM 3193 OH2 TIP3 3474 7.662 -18.594 11.356 -0.834 1.520 -ATOM 3194 H1 TIP3 3474 8.156 -18.449 10.505 0.417 1.000 -ATOM 3195 H2 TIP3 3474 8.399 -18.468 12.020 0.417 1.000 -ATOM 3196 OH2 TIP3 3476 10.937 -19.461 3.258 -0.834 1.520 -ATOM 3197 H1 TIP3 3476 11.561 -18.920 2.840 0.417 1.000 -ATOM 3198 H2 TIP3 3476 10.746 -19.011 4.107 0.417 1.000 -ATOM 3199 OH2 TIP3 3484 6.929 -17.058 -8.449 -0.834 1.520 -ATOM 3200 H1 TIP3 3484 7.470 -17.669 -8.955 0.417 1.000 -ATOM 3201 H2 TIP3 3484 6.250 -17.632 -8.048 0.417 1.000 -ATOM 3202 OH2 TIP3 3496 -1.365 -14.560 17.484 -0.834 1.520 -ATOM 3203 H1 TIP3 3496 -0.699 -13.859 17.377 0.417 1.000 -ATOM 3204 H2 TIP3 3496 -2.066 -14.222 16.946 0.417 1.000 -ATOM 3205 OH2 TIP3 3511 15.030 -9.953 -19.026 -0.834 1.520 -ATOM 3206 H1 TIP3 3511 15.212 -9.882 -18.080 0.417 1.000 -ATOM 3207 H2 TIP3 3511 14.099 -10.210 -19.053 0.417 1.000 -ATOM 3208 OH2 TIP3 3513 11.120 -7.431 -0.992 -0.834 1.520 -ATOM 3209 H1 TIP3 3513 11.055 -6.939 -0.104 0.417 1.000 -ATOM 3210 H2 TIP3 3513 11.846 -7.013 -1.468 0.417 1.000 -ATOM 3211 OH2 TIP3 3526 0.597 -5.368 -18.938 -0.834 1.520 -ATOM 3212 H1 TIP3 3526 1.521 -5.203 -18.586 0.417 1.000 -ATOM 3213 H2 TIP3 3526 0.147 -5.910 -18.280 0.417 1.000 -ATOM 3214 OH2 TIP3 3527 -1.662 -9.030 -9.881 -0.834 1.520 -ATOM 3215 H1 TIP3 3527 -0.912 -9.437 -9.468 0.417 1.000 -ATOM 3216 H2 TIP3 3527 -1.689 -9.409 -10.783 0.417 1.000 -ATOM 3217 OH2 TIP3 3533 16.855 -16.550 -15.932 -0.834 1.520 -ATOM 3218 H1 TIP3 3533 16.377 -17.127 -16.496 0.417 1.000 -ATOM 3219 H2 TIP3 3533 17.457 -16.152 -16.610 0.417 1.000 -ATOM 3220 OH2 TIP3 3534 1.664 -8.200 -7.213 -0.834 1.520 -ATOM 3221 H1 TIP3 3534 1.444 -7.476 -7.781 0.417 1.000 -ATOM 3222 H2 TIP3 3534 2.536 -8.468 -7.528 0.417 1.000 -ATOM 3223 OH2 TIP3 3537 -6.172 -9.878 13.931 -0.834 1.520 -ATOM 3224 H1 TIP3 3537 -6.340 -9.113 14.448 0.417 1.000 -ATOM 3225 H2 TIP3 3537 -6.555 -9.618 13.104 0.417 1.000 -ATOM 3226 OH2 TIP3 3545 -9.593 -13.351 -11.112 -0.834 1.520 -ATOM 3227 H1 TIP3 3545 -10.190 -12.785 -10.614 0.417 1.000 -ATOM 3228 H2 TIP3 3545 -8.923 -13.583 -10.453 0.417 1.000 -ATOM 3229 OH2 TIP3 3548 -2.183 -7.240 -12.578 -0.834 1.520 -ATOM 3230 H1 TIP3 3548 -1.894 -7.340 -11.612 0.417 1.000 -ATOM 3231 H2 TIP3 3548 -2.840 -6.517 -12.553 0.417 1.000 -ATOM 3232 OH2 TIP3 3554 -0.662 -10.156 -12.476 -0.834 1.520 -ATOM 3233 H1 TIP3 3554 -0.464 -10.789 -13.212 0.417 1.000 -ATOM 3234 H2 TIP3 3554 0.226 -9.867 -12.171 0.417 1.000 -ATOM 3235 OH2 TIP3 3555 8.615 -15.272 -3.578 -0.834 1.520 -ATOM 3236 H1 TIP3 3555 8.225 -16.019 -4.153 0.417 1.000 -ATOM 3237 H2 TIP3 3555 9.456 -15.164 -4.049 0.417 1.000 -ATOM 3238 OH2 TIP3 3556 1.245 -11.884 3.968 -0.834 1.520 -ATOM 3239 H1 TIP3 3556 1.257 -11.460 4.803 0.417 1.000 -ATOM 3240 H2 TIP3 3556 0.394 -12.328 3.868 0.417 1.000 -ATOM 3241 OH2 TIP3 3557 1.166 -4.178 6.322 -0.834 1.520 -ATOM 3242 H1 TIP3 3557 0.234 -3.995 6.686 0.417 1.000 -ATOM 3243 H2 TIP3 3557 1.754 -3.895 7.059 0.417 1.000 -ATOM 3244 OH2 TIP3 3558 9.568 -9.476 11.315 -0.834 1.520 -ATOM 3245 H1 TIP3 3558 9.760 -9.233 10.405 0.417 1.000 -ATOM 3246 H2 TIP3 3558 9.048 -10.308 11.270 0.417 1.000 -ATOM 3247 OH2 TIP3 3561 12.369 -14.725 10.147 -0.834 1.520 -ATOM 3248 H1 TIP3 3561 11.932 -13.978 9.735 0.417 1.000 -ATOM 3249 H2 TIP3 3561 13.298 -14.623 9.839 0.417 1.000 -ATOM 3250 OH2 TIP3 3563 14.432 -13.795 16.075 -0.834 1.520 -ATOM 3251 H1 TIP3 3563 15.229 -13.972 15.624 0.417 1.000 -ATOM 3252 H2 TIP3 3563 14.534 -12.793 16.324 0.417 1.000 -ATOM 3253 OH2 TIP3 3569 8.012 -14.434 -10.411 -0.834 1.520 -ATOM 3254 H1 TIP3 3569 8.394 -15.312 -10.312 0.417 1.000 -ATOM 3255 H2 TIP3 3569 8.330 -14.152 -11.287 0.417 1.000 -ATOM 3256 OH2 TIP3 3573 5.331 -10.120 -15.621 -0.834 1.520 -ATOM 3257 H1 TIP3 3573 5.003 -10.588 -16.448 0.417 1.000 -ATOM 3258 H2 TIP3 3573 5.326 -10.861 -14.956 0.417 1.000 -ATOM 3259 OH2 TIP3 3575 0.692 -1.258 -7.867 -0.834 1.520 -ATOM 3260 H1 TIP3 3575 0.174 -1.419 -7.053 0.417 1.000 -ATOM 3261 H2 TIP3 3575 0.895 -2.180 -8.145 0.417 1.000 -ATOM 3262 OH2 TIP3 3576 8.824 -10.422 14.277 -0.834 1.520 -ATOM 3263 H1 TIP3 3576 9.395 -11.161 14.092 0.417 1.000 -ATOM 3264 H2 TIP3 3576 8.903 -9.892 13.539 0.417 1.000 -ATOM 3265 OH2 TIP3 3579 12.304 -14.975 12.794 -0.834 1.520 -ATOM 3266 H1 TIP3 3579 13.262 -14.898 13.108 0.417 1.000 -ATOM 3267 H2 TIP3 3579 12.441 -15.135 11.819 0.417 1.000 -ATOM 3268 OH2 TIP3 3580 3.911 -3.110 16.592 -0.834 1.520 -ATOM 3269 H1 TIP3 3580 4.082 -3.804 15.963 0.417 1.000 -ATOM 3270 H2 TIP3 3580 3.767 -3.685 17.335 0.417 1.000 -ATOM 3271 OH2 TIP3 3582 12.381 -6.189 13.041 -0.834 1.520 -ATOM 3272 H1 TIP3 3582 12.766 -5.332 13.086 0.417 1.000 -ATOM 3273 H2 TIP3 3582 12.213 -6.361 13.965 0.417 1.000 -ATOM 3274 OH2 TIP3 3586 1.378 -6.976 18.454 -0.834 1.520 -ATOM 3275 H1 TIP3 3586 0.547 -7.171 17.981 0.417 1.000 -ATOM 3276 H2 TIP3 3586 1.598 -7.782 18.914 0.417 1.000 -ATOM 3277 OH2 TIP3 3592 -1.489 17.143 -11.665 -0.834 1.520 -ATOM 3278 H1 TIP3 3592 -0.930 16.307 -11.849 0.417 1.000 -ATOM 3279 H2 TIP3 3592 -1.793 16.991 -10.695 0.417 1.000 -ATOM 3280 OH2 TIP3 3593 12.013 -7.181 -18.277 -0.834 1.520 -ATOM 3281 H1 TIP3 3593 12.460 -6.480 -17.745 0.417 1.000 -ATOM 3282 H2 TIP3 3593 11.701 -7.823 -17.646 0.417 1.000 -ATOM 3283 OH2 TIP3 3595 3.999 -5.659 -8.781 -0.834 1.520 -ATOM 3284 H1 TIP3 3595 3.278 -5.411 -9.279 0.417 1.000 -ATOM 3285 H2 TIP3 3595 4.589 -4.919 -8.951 0.417 1.000 -ATOM 3286 OH2 TIP3 3596 -12.426 -0.580 -15.283 -0.834 1.520 -ATOM 3287 H1 TIP3 3596 -12.296 -0.665 -16.321 0.417 1.000 -ATOM 3288 H2 TIP3 3596 -12.030 0.255 -15.043 0.417 1.000 -ATOM 3289 OH2 TIP3 3597 -0.426 0.666 -9.283 -0.834 1.520 -ATOM 3290 H1 TIP3 3597 0.069 -0.098 -8.966 0.417 1.000 -ATOM 3291 H2 TIP3 3597 -1.173 0.684 -8.672 0.417 1.000 -ATOM 3292 OH2 TIP3 3599 6.427 -13.259 -0.696 -0.834 1.520 -ATOM 3293 H1 TIP3 3599 5.883 -13.835 -0.134 0.417 1.000 -ATOM 3294 H2 TIP3 3599 6.265 -12.402 -0.281 0.417 1.000 -ATOM 3295 OH2 TIP3 3600 3.864 0.095 5.624 -0.834 1.520 -ATOM 3296 H1 TIP3 3600 3.843 -0.333 6.488 0.417 1.000 -ATOM 3297 H2 TIP3 3600 4.751 0.351 5.601 0.417 1.000 -ATOM 3298 OH2 TIP3 3601 6.333 -0.644 10.407 -0.834 1.520 -ATOM 3299 H1 TIP3 3601 5.560 -0.189 9.912 0.417 1.000 -ATOM 3300 H2 TIP3 3601 6.320 -0.091 11.252 0.417 1.000 -ATOM 3301 OH2 TIP3 3603 0.844 -9.177 7.657 -0.834 1.520 -ATOM 3302 H1 TIP3 3603 0.261 -9.923 7.795 0.417 1.000 -ATOM 3303 H2 TIP3 3603 0.406 -8.447 8.084 0.417 1.000 -ATOM 3304 OH2 TIP3 3604 10.960 -5.317 17.453 -0.834 1.520 -ATOM 3305 H1 TIP3 3604 11.745 -5.605 16.912 0.417 1.000 -ATOM 3306 H2 TIP3 3604 10.257 -5.140 16.781 0.417 1.000 -ATOM 3307 OH2 TIP3 3612 7.212 -14.449 -18.038 -0.834 1.520 -ATOM 3308 H1 TIP3 3612 6.802 -14.880 -17.376 0.417 1.000 -ATOM 3309 H2 TIP3 3612 7.823 -13.884 -17.577 0.417 1.000 -ATOM 3310 OH2 TIP3 3614 3.131 -1.015 -6.319 -0.834 1.520 -ATOM 3311 H1 TIP3 3614 2.287 -1.216 -6.795 0.417 1.000 -ATOM 3312 H2 TIP3 3614 3.593 -1.855 -6.348 0.417 1.000 -ATOM 3313 OH2 TIP3 3615 4.227 -3.502 -6.350 -0.834 1.520 -ATOM 3314 H1 TIP3 3615 4.060 -4.209 -5.784 0.417 1.000 -ATOM 3315 H2 TIP3 3615 4.818 -3.853 -7.021 0.417 1.000 -ATOM 3316 OH2 TIP3 3617 9.059 10.470 6.160 -0.834 1.520 -ATOM 3317 H1 TIP3 3617 9.043 10.109 7.068 0.417 1.000 -ATOM 3318 H2 TIP3 3617 8.226 10.093 5.792 0.417 1.000 -ATOM 3319 OH2 TIP3 3618 -9.550 -4.558 7.143 -0.834 1.520 -ATOM 3320 H1 TIP3 3618 -9.362 -4.757 8.031 0.417 1.000 -ATOM 3321 H2 TIP3 3618 -9.376 -3.591 7.099 0.417 1.000 -ATOM 3322 OH2 TIP3 3619 -2.063 -15.343 -8.797 -0.834 1.520 -ATOM 3323 H1 TIP3 3619 -2.082 -16.078 -8.121 0.417 1.000 -ATOM 3324 H2 TIP3 3619 -2.296 -15.827 -9.585 0.417 1.000 -ATOM 3325 OH2 TIP3 3620 3.458 -8.491 8.295 -0.834 1.520 -ATOM 3326 H1 TIP3 3620 3.834 -9.124 7.676 0.417 1.000 -ATOM 3327 H2 TIP3 3620 2.483 -8.681 8.283 0.417 1.000 -ATOM 3328 OH2 TIP3 3621 12.515 -3.794 6.473 -0.834 1.520 -ATOM 3329 H1 TIP3 3621 12.822 -4.094 7.292 0.417 1.000 -ATOM 3330 H2 TIP3 3621 12.601 -2.845 6.532 0.417 1.000 -ATOM 3331 OH2 TIP3 3622 11.843 -19.549 11.115 -0.834 1.520 -ATOM 3332 H1 TIP3 3622 12.030 -19.834 11.998 0.417 1.000 -ATOM 3333 H2 TIP3 3622 12.739 -19.295 10.767 0.417 1.000 -ATOM 3334 OH2 TIP3 3623 11.529 -13.787 -4.755 -0.834 1.520 -ATOM 3335 H1 TIP3 3623 11.986 -14.649 -4.637 0.417 1.000 -ATOM 3336 H2 TIP3 3623 11.168 -13.887 -5.657 0.417 1.000 -ATOM 3337 OH2 TIP3 3625 -0.068 10.597 10.791 -0.834 1.520 -ATOM 3338 H1 TIP3 3625 -0.133 10.286 11.647 0.417 1.000 -ATOM 3339 H2 TIP3 3625 -0.870 10.352 10.345 0.417 1.000 -ATOM 3340 OH2 TIP3 3626 0.151 -10.032 12.291 -0.834 1.520 -ATOM 3341 H1 TIP3 3626 0.891 -10.666 12.484 0.417 1.000 -ATOM 3342 H2 TIP3 3626 0.624 -9.166 12.268 0.417 1.000 -ATOM 3343 OH2 TIP3 3627 -2.624 -8.100 18.779 -0.834 1.520 -ATOM 3344 H1 TIP3 3627 -1.963 -8.189 19.498 0.417 1.000 -ATOM 3345 H2 TIP3 3627 -3.193 -8.844 18.913 0.417 1.000 -ATOM 3346 OH2 TIP3 3628 1.703 -2.991 14.150 -0.834 1.520 -ATOM 3347 H1 TIP3 3628 2.047 -3.918 14.071 0.417 1.000 -ATOM 3348 H2 TIP3 3628 2.343 -2.490 13.690 0.417 1.000 -ATOM 3349 OH2 TIP3 3633 7.510 2.709 -10.593 -0.834 1.520 -ATOM 3350 H1 TIP3 3633 8.180 2.143 -10.210 0.417 1.000 -ATOM 3351 H2 TIP3 3633 8.048 3.575 -10.668 0.417 1.000 -ATOM 3352 OH2 TIP3 3634 8.834 9.646 -3.470 -0.834 1.520 -ATOM 3353 H1 TIP3 3634 8.945 9.168 -2.616 0.417 1.000 -ATOM 3354 H2 TIP3 3634 7.869 9.694 -3.620 0.417 1.000 -ATOM 3355 OH2 TIP3 3636 10.680 3.011 -16.453 -0.834 1.520 -ATOM 3356 H1 TIP3 3636 10.863 3.667 -15.780 0.417 1.000 -ATOM 3357 H2 TIP3 3636 10.122 2.449 -15.860 0.417 1.000 -ATOM 3358 OH2 TIP3 3638 9.991 8.121 -12.768 -0.834 1.520 -ATOM 3359 H1 TIP3 3638 9.530 8.721 -12.161 0.417 1.000 -ATOM 3360 H2 TIP3 3638 10.596 7.688 -12.130 0.417 1.000 -ATOM 3361 OH2 TIP3 3639 9.518 -0.807 1.553 -0.834 1.520 -ATOM 3362 H1 TIP3 3639 8.577 -0.843 1.744 0.417 1.000 -ATOM 3363 H2 TIP3 3639 9.949 -1.501 2.148 0.417 1.000 -ATOM 3364 OH2 TIP3 3640 -1.059 3.457 10.901 -0.834 1.520 -ATOM 3365 H1 TIP3 3640 -1.658 4.220 10.616 0.417 1.000 -ATOM 3366 H2 TIP3 3640 -0.697 3.144 10.063 0.417 1.000 -ATOM 3367 OH2 TIP3 3641 -1.605 -4.534 -5.785 -0.834 1.520 -ATOM 3368 H1 TIP3 3641 -0.751 -5.012 -5.934 0.417 1.000 -ATOM 3369 H2 TIP3 3641 -1.957 -4.900 -4.975 0.417 1.000 -ATOM 3370 OH2 TIP3 3642 6.227 -9.871 7.843 -0.834 1.520 -ATOM 3371 H1 TIP3 3642 6.036 -8.980 7.661 0.417 1.000 -ATOM 3372 H2 TIP3 3642 7.022 -10.012 7.355 0.417 1.000 -ATOM 3373 OH2 TIP3 3643 5.868 -10.137 11.056 -0.834 1.520 -ATOM 3374 H1 TIP3 3643 5.641 -9.497 10.437 0.417 1.000 -ATOM 3375 H2 TIP3 3643 5.435 -9.850 11.915 0.417 1.000 -ATOM 3376 OH2 TIP3 3650 -7.746 -0.324 -17.771 -0.834 1.520 -ATOM 3377 H1 TIP3 3650 -7.322 0.538 -17.642 0.417 1.000 -ATOM 3378 H2 TIP3 3650 -7.885 -0.465 -18.750 0.417 1.000 -ATOM 3379 OH2 TIP3 3651 -3.114 3.849 -18.438 -0.834 1.520 -ATOM 3380 H1 TIP3 3651 -2.274 4.125 -18.958 0.417 1.000 -ATOM 3381 H2 TIP3 3651 -3.749 4.508 -18.750 0.417 1.000 -ATOM 3382 OH2 TIP3 3653 0.421 -11.489 -5.429 -0.834 1.520 -ATOM 3383 H1 TIP3 3653 0.722 -11.750 -4.559 0.417 1.000 -ATOM 3384 H2 TIP3 3653 -0.085 -10.715 -5.240 0.417 1.000 -ATOM 3385 OH2 TIP3 3657 13.560 -4.859 2.959 -0.834 1.520 -ATOM 3386 H1 TIP3 3657 13.604 -3.944 3.209 0.417 1.000 -ATOM 3387 H2 TIP3 3657 13.626 -5.397 3.811 0.417 1.000 -ATOM 3388 OH2 TIP3 3658 13.349 -2.064 3.595 -0.834 1.520 -ATOM 3389 H1 TIP3 3658 13.408 -1.620 4.471 0.417 1.000 -ATOM 3390 H2 TIP3 3658 14.053 -1.597 3.056 0.417 1.000 -ATOM 3391 OH2 TIP3 3659 -2.799 12.642 -4.810 -0.834 1.520 -ATOM 3392 H1 TIP3 3659 -2.209 13.341 -5.189 0.417 1.000 -ATOM 3393 H2 TIP3 3659 -2.245 12.224 -4.113 0.417 1.000 -ATOM 3394 OH2 TIP3 3660 -5.964 -12.270 -13.356 -0.834 1.520 -ATOM 3395 H1 TIP3 3660 -5.642 -12.862 -12.654 0.417 1.000 -ATOM 3396 H2 TIP3 3660 -6.114 -11.411 -12.877 0.417 1.000 -ATOM 3397 OH2 TIP3 3661 -11.976 3.598 -2.565 -0.834 1.520 -ATOM 3398 H1 TIP3 3661 -11.098 3.477 -2.956 0.417 1.000 -ATOM 3399 H2 TIP3 3661 -12.350 4.418 -2.883 0.417 1.000 -ATOM 3400 OH2 TIP3 3662 -1.094 -9.315 -4.907 -0.834 1.520 -ATOM 3401 H1 TIP3 3662 -1.480 -8.805 -5.617 0.417 1.000 -ATOM 3402 H2 TIP3 3662 -1.727 -10.004 -4.728 0.417 1.000 -ATOM 3403 OH2 TIP3 3663 15.988 -1.510 15.037 -0.834 1.520 -ATOM 3404 H1 TIP3 3663 15.252 -1.110 15.531 0.417 1.000 -ATOM 3405 H2 TIP3 3663 16.749 -1.311 15.667 0.417 1.000 -ATOM 3406 OH2 TIP3 3664 16.697 11.319 7.188 -0.834 1.520 -ATOM 3407 H1 TIP3 3664 17.468 11.942 6.968 0.417 1.000 -ATOM 3408 H2 TIP3 3664 16.113 11.982 7.478 0.417 1.000 -ATOM 3409 OH2 TIP3 3665 -16.498 -1.262 6.015 -0.834 1.520 -ATOM 3410 H1 TIP3 3665 -15.958 -0.566 6.414 0.417 1.000 -ATOM 3411 H2 TIP3 3665 -16.785 -0.812 5.221 0.417 1.000 -ATOM 3412 OH2 TIP3 3674 10.470 -1.289 -16.235 -0.834 1.520 -ATOM 3413 H1 TIP3 3674 11.181 -1.546 -16.789 0.417 1.000 -ATOM 3414 H2 TIP3 3674 9.687 -1.279 -16.766 0.417 1.000 -ATOM 3415 OH2 TIP3 3675 5.926 9.198 -3.389 -0.834 1.520 -ATOM 3416 H1 TIP3 3675 5.471 8.950 -2.552 0.417 1.000 -ATOM 3417 H2 TIP3 3675 6.067 8.349 -3.828 0.417 1.000 -ATOM 3418 OH2 TIP3 3677 -1.703 13.419 -13.656 -0.834 1.520 -ATOM 3419 H1 TIP3 3677 -1.947 14.034 -14.380 0.417 1.000 -ATOM 3420 H2 TIP3 3677 -0.938 13.788 -13.230 0.417 1.000 -ATOM 3421 OH2 TIP3 3678 14.802 2.107 6.297 -0.834 1.520 -ATOM 3422 H1 TIP3 3678 13.985 2.571 6.115 0.417 1.000 -ATOM 3423 H2 TIP3 3678 15.066 2.359 7.253 0.417 1.000 -ATOM 3424 OH2 TIP3 3680 10.034 -5.650 5.943 -0.834 1.520 -ATOM 3425 H1 TIP3 3680 9.652 -4.714 6.018 0.417 1.000 -ATOM 3426 H2 TIP3 3680 10.910 -5.497 6.214 0.417 1.000 -ATOM 3427 OH2 TIP3 3681 12.093 18.549 -19.160 -0.834 1.520 -ATOM 3428 H1 TIP3 3681 12.906 18.958 -18.839 0.417 1.000 -ATOM 3429 H2 TIP3 3681 11.546 18.598 -18.361 0.417 1.000 -ATOM 3430 OH2 TIP3 3682 8.651 -7.404 2.449 -0.834 1.520 -ATOM 3431 H1 TIP3 3682 9.290 -7.786 3.072 0.417 1.000 -ATOM 3432 H2 TIP3 3682 7.869 -7.890 2.508 0.417 1.000 -ATOM 3433 OH2 TIP3 3683 -8.822 16.778 18.226 -0.834 1.520 -ATOM 3434 H1 TIP3 3683 -8.780 15.842 18.508 0.417 1.000 -ATOM 3435 H2 TIP3 3683 -7.966 17.155 18.555 0.417 1.000 -ATOM 3436 OH2 TIP3 3684 6.505 0.722 6.381 -0.834 1.520 -ATOM 3437 H1 TIP3 3684 7.423 0.915 6.251 0.417 1.000 -ATOM 3438 H2 TIP3 3684 6.508 0.422 7.279 0.417 1.000 -ATOM 3439 OH2 TIP3 3685 -2.028 6.015 14.219 -0.834 1.520 -ATOM 3440 H1 TIP3 3685 -2.554 5.249 14.270 0.417 1.000 -ATOM 3441 H2 TIP3 3685 -2.631 6.802 13.987 0.417 1.000 -ATOM 3442 OH2 TIP3 3686 18.803 0.448 19.090 -0.834 1.520 -ATOM 3443 H1 TIP3 3686 17.883 0.432 19.103 0.417 1.000 -ATOM 3444 H2 TIP3 3686 19.035 -0.371 19.547 0.417 1.000 -ATOM 3445 OH2 TIP3 3687 -14.094 12.297 15.892 -0.834 1.520 -ATOM 3446 H1 TIP3 3687 -14.111 11.416 16.358 0.417 1.000 -ATOM 3447 H2 TIP3 3687 -14.296 12.007 14.983 0.417 1.000 -ATOM 3448 OH2 TIP3 3688 4.952 6.251 6.578 -0.834 1.520 -ATOM 3449 H1 TIP3 3688 4.751 7.051 6.086 0.417 1.000 -ATOM 3450 H2 TIP3 3688 4.062 6.040 6.932 0.417 1.000 -ATOM 3451 OH2 TIP3 3693 8.780 6.524 -17.444 -0.834 1.520 -ATOM 3452 H1 TIP3 3693 9.484 7.192 -17.481 0.417 1.000 -ATOM 3453 H2 TIP3 3693 8.952 6.125 -16.582 0.417 1.000 -ATOM 3454 OH2 TIP3 3697 0.584 9.083 -10.742 -0.834 1.520 -ATOM 3455 H1 TIP3 3697 0.206 9.670 -9.996 0.417 1.000 -ATOM 3456 H2 TIP3 3697 1.495 9.378 -10.905 0.417 1.000 -ATOM 3457 OH2 TIP3 3698 4.169 11.178 -12.907 -0.834 1.520 -ATOM 3458 H1 TIP3 3698 4.122 11.962 -13.524 0.417 1.000 -ATOM 3459 H2 TIP3 3698 3.761 10.528 -13.519 0.417 1.000 -ATOM 3460 OH2 TIP3 3700 13.724 6.470 -8.627 -0.834 1.520 -ATOM 3461 H1 TIP3 3700 13.880 5.569 -8.976 0.417 1.000 -ATOM 3462 H2 TIP3 3700 13.190 6.279 -7.832 0.417 1.000 -ATOM 3463 OH2 TIP3 3701 15.573 0.568 -7.601 -0.834 1.520 -ATOM 3464 H1 TIP3 3701 15.495 -0.091 -8.311 0.417 1.000 -ATOM 3465 H2 TIP3 3701 16.536 0.645 -7.501 0.417 1.000 -ATOM 3466 OH2 TIP3 3702 -10.840 15.087 -2.911 -0.834 1.520 -ATOM 3467 H1 TIP3 3702 -11.107 14.303 -3.482 0.417 1.000 -ATOM 3468 H2 TIP3 3702 -11.489 15.788 -3.081 0.417 1.000 -ATOM 3469 OH2 TIP3 3703 0.942 4.743 1.022 -0.834 1.520 -ATOM 3470 H1 TIP3 3703 0.045 4.484 0.827 0.417 1.000 -ATOM 3471 H2 TIP3 3703 0.951 5.726 0.788 0.417 1.000 -ATOM 3472 OH2 TIP3 3704 -7.979 -2.156 11.021 -0.834 1.520 -ATOM 3473 H1 TIP3 3704 -7.608 -1.226 11.010 0.417 1.000 -ATOM 3474 H2 TIP3 3704 -8.319 -2.211 11.918 0.417 1.000 -ATOM 3475 OH2 TIP3 3705 13.481 13.909 0.236 -0.834 1.520 -ATOM 3476 H1 TIP3 3705 13.407 13.374 1.086 0.417 1.000 -ATOM 3477 H2 TIP3 3705 13.571 13.236 -0.419 0.417 1.000 -ATOM 3478 OH2 TIP3 3706 -2.728 13.927 9.206 -0.834 1.520 -ATOM 3479 H1 TIP3 3706 -2.543 13.467 8.331 0.417 1.000 -ATOM 3480 H2 TIP3 3706 -3.644 13.784 9.311 0.417 1.000 -ATOM 3481 OH2 TIP3 3707 7.535 5.870 5.374 -0.834 1.520 -ATOM 3482 H1 TIP3 3707 7.116 5.050 5.042 0.417 1.000 -ATOM 3483 H2 TIP3 3707 6.762 6.449 5.486 0.417 1.000 -ATOM 3484 OH2 TIP3 3708 -3.740 7.219 1.916 -0.834 1.520 -ATOM 3485 H1 TIP3 3708 -4.489 6.653 1.903 0.417 1.000 -ATOM 3486 H2 TIP3 3708 -3.649 7.594 0.992 0.417 1.000 -ATOM 3487 OH2 TIP3 3709 16.983 10.438 17.869 -0.834 1.520 -ATOM 3488 H1 TIP3 3709 16.826 9.559 18.168 0.417 1.000 -ATOM 3489 H2 TIP3 3709 16.427 10.984 18.393 0.417 1.000 -ATOM 3490 OH2 TIP3 3710 19.749 3.431 12.417 -0.834 1.520 -ATOM 3491 H1 TIP3 3710 19.132 2.941 12.963 0.417 1.000 -ATOM 3492 H2 TIP3 3710 19.514 3.210 11.550 0.417 1.000 -ATOM 3493 OH2 TIP3 3718 -0.356 4.996 -19.477 -0.834 1.520 -ATOM 3494 H1 TIP3 3718 -0.243 5.535 -18.635 0.417 1.000 -ATOM 3495 H2 TIP3 3718 0.557 4.741 -19.711 0.417 1.000 -ATOM 3496 OH2 TIP3 3719 12.066 13.024 -17.447 -0.834 1.520 -ATOM 3497 H1 TIP3 3719 11.853 13.167 -16.490 0.417 1.000 -ATOM 3498 H2 TIP3 3719 11.146 12.995 -17.901 0.417 1.000 -ATOM 3499 OH2 TIP3 3720 3.390 9.506 -6.870 -0.834 1.520 -ATOM 3500 H1 TIP3 3720 4.282 9.326 -7.288 0.417 1.000 -ATOM 3501 H2 TIP3 3720 2.986 10.210 -7.391 0.417 1.000 -ATOM 3502 OH2 TIP3 3722 8.287 9.686 -11.030 -0.834 1.520 -ATOM 3503 H1 TIP3 3722 8.048 9.997 -10.130 0.417 1.000 -ATOM 3504 H2 TIP3 3722 7.596 8.964 -11.101 0.417 1.000 -ATOM 3505 OH2 TIP3 3723 -8.666 6.903 -8.696 -0.834 1.520 -ATOM 3506 H1 TIP3 3723 -9.233 7.696 -8.424 0.417 1.000 -ATOM 3507 H2 TIP3 3723 -8.209 6.796 -7.911 0.417 1.000 -ATOM 3508 OH2 TIP3 3724 5.139 10.800 2.203 -0.834 1.520 -ATOM 3509 H1 TIP3 3724 5.778 10.185 2.537 0.417 1.000 -ATOM 3510 H2 TIP3 3724 4.555 10.169 1.700 0.417 1.000 -ATOM 3511 OH2 TIP3 3725 5.658 11.400 7.140 -0.834 1.520 -ATOM 3512 H1 TIP3 3725 4.862 11.443 7.677 0.417 1.000 -ATOM 3513 H2 TIP3 3725 6.368 11.246 7.826 0.417 1.000 -ATOM 3514 OH2 TIP3 3727 17.428 1.222 -1.442 -0.834 1.520 -ATOM 3515 H1 TIP3 3727 17.988 1.972 -1.282 0.417 1.000 -ATOM 3516 H2 TIP3 3727 16.732 1.392 -0.820 0.417 1.000 -ATOM 3517 OH2 TIP3 3728 -12.129 5.057 4.016 -0.834 1.520 -ATOM 3518 H1 TIP3 3728 -12.639 5.718 3.480 0.417 1.000 -ATOM 3519 H2 TIP3 3728 -11.901 4.344 3.402 0.417 1.000 -ATOM 3520 OH2 TIP3 3729 8.465 -0.262 19.336 -0.834 1.520 -ATOM 3521 H1 TIP3 3729 7.950 -0.884 19.864 0.417 1.000 -ATOM 3522 H2 TIP3 3729 8.248 -0.539 18.432 0.417 1.000 -ATOM 3523 OH2 TIP3 3730 14.434 16.027 18.498 -0.834 1.520 -ATOM 3524 H1 TIP3 3730 15.301 15.898 18.847 0.417 1.000 -ATOM 3525 H2 TIP3 3730 14.103 16.711 19.082 0.417 1.000 -ATOM 3526 OH2 TIP3 3731 0.523 11.177 15.364 -0.834 1.520 -ATOM 3527 H1 TIP3 3731 0.219 12.049 15.727 0.417 1.000 -ATOM 3528 H2 TIP3 3731 1.480 11.276 15.419 0.417 1.000 -ATOM 3529 OH2 TIP3 3736 -4.840 -3.683 17.828 -0.834 1.520 -ATOM 3530 H1 TIP3 3736 -4.151 -4.249 18.258 0.417 1.000 -ATOM 3531 H2 TIP3 3736 -4.328 -3.179 17.194 0.417 1.000 -ATOM 3532 OH2 TIP3 3737 -2.150 18.594 -16.699 -0.834 1.520 -ATOM 3533 H1 TIP3 3737 -2.657 19.365 -17.033 0.417 1.000 -ATOM 3534 H2 TIP3 3737 -2.801 17.884 -16.797 0.417 1.000 -ATOM 3535 OH2 TIP3 3740 3.323 9.785 -10.197 -0.834 1.520 -ATOM 3536 H1 TIP3 3740 4.151 10.268 -10.381 0.417 1.000 -ATOM 3537 H2 TIP3 3740 3.301 9.193 -10.864 0.417 1.000 -ATOM 3538 OH2 TIP3 3743 6.993 15.288 0.879 -0.834 1.520 -ATOM 3539 H1 TIP3 3743 6.493 16.117 1.214 0.417 1.000 -ATOM 3540 H2 TIP3 3743 6.584 14.547 1.322 0.417 1.000 -ATOM 3541 OH2 TIP3 3745 -2.593 5.737 5.400 -0.834 1.520 -ATOM 3542 H1 TIP3 3745 -2.392 5.784 4.485 0.417 1.000 -ATOM 3543 H2 TIP3 3745 -1.997 5.061 5.697 0.417 1.000 -ATOM 3544 OH2 TIP3 3746 -1.845 16.925 9.762 -0.834 1.520 -ATOM 3545 H1 TIP3 3746 -2.238 16.448 9.069 0.417 1.000 -ATOM 3546 H2 TIP3 3746 -1.293 17.585 9.292 0.417 1.000 -ATOM 3547 OH2 TIP3 3747 11.049 14.599 4.966 -0.834 1.520 -ATOM 3548 H1 TIP3 3747 10.611 15.284 5.511 0.417 1.000 -ATOM 3549 H2 TIP3 3747 10.546 14.658 4.099 0.417 1.000 -ATOM 3550 OH2 TIP3 3749 7.095 13.829 9.788 -0.834 1.520 -ATOM 3551 H1 TIP3 3749 6.154 14.022 9.709 0.417 1.000 -ATOM 3552 H2 TIP3 3749 7.132 12.848 9.690 0.417 1.000 -ATOM 3553 OH2 TIP3 3751 16.188 -1.598 19.026 -0.834 1.520 -ATOM 3554 H1 TIP3 3751 16.729 -1.821 19.805 0.417 1.000 -ATOM 3555 H2 TIP3 3751 15.286 -1.683 19.326 0.417 1.000 -ATOM 3556 OH2 TIP3 3753 -4.888 9.781 18.252 -0.834 1.520 -ATOM 3557 H1 TIP3 3753 -4.108 9.858 17.589 0.417 1.000 -ATOM 3558 H2 TIP3 3753 -4.441 9.570 19.142 0.417 1.000 -ATOM 3559 OH2 TIP3 3757 13.974 2.808 -17.829 -0.834 1.520 -ATOM 3560 H1 TIP3 3757 14.889 3.172 -17.879 0.417 1.000 -ATOM 3561 H2 TIP3 3757 13.813 2.805 -16.890 0.417 1.000 -ATOM 3562 OH2 TIP3 3761 -0.892 11.667 -2.919 -0.834 1.520 -ATOM 3563 H1 TIP3 3761 -1.396 11.241 -2.230 0.417 1.000 -ATOM 3564 H2 TIP3 3761 -0.380 12.420 -2.436 0.417 1.000 -ATOM 3565 OH2 TIP3 3762 12.961 16.077 -17.240 -0.834 1.520 -ATOM 3566 H1 TIP3 3762 13.235 15.952 -16.317 0.417 1.000 -ATOM 3567 H2 TIP3 3762 11.991 15.792 -17.268 0.417 1.000 -ATOM 3568 OH2 TIP3 3764 -8.006 15.054 -13.253 -0.834 1.520 -ATOM 3569 H1 TIP3 3764 -7.587 15.009 -14.151 0.417 1.000 -ATOM 3570 H2 TIP3 3764 -7.656 15.913 -12.919 0.417 1.000 -ATOM 3571 OH2 TIP3 3766 6.321 12.780 12.842 -0.834 1.520 -ATOM 3572 H1 TIP3 3766 5.775 12.338 12.142 0.417 1.000 -ATOM 3573 H2 TIP3 3766 6.386 13.680 12.475 0.417 1.000 -ATOM 3574 OH2 TIP3 3767 5.196 17.913 11.742 -0.834 1.520 -ATOM 3575 H1 TIP3 3767 6.056 18.105 12.100 0.417 1.000 -ATOM 3576 H2 TIP3 3767 4.976 17.080 12.265 0.417 1.000 -ATOM 3577 OH2 TIP3 3770 10.039 12.865 1.378 -0.834 1.520 -ATOM 3578 H1 TIP3 3770 9.710 12.646 0.494 0.417 1.000 -ATOM 3579 H2 TIP3 3770 9.669 13.778 1.524 0.417 1.000 -ATOM 3580 OH2 TIP3 3771 10.814 15.270 15.396 -0.834 1.520 -ATOM 3581 H1 TIP3 3771 10.991 15.798 14.658 0.417 1.000 -ATOM 3582 H2 TIP3 3771 11.342 15.731 16.033 0.417 1.000 -ATOM 3583 OH2 TIP3 3776 0.228 17.963 -18.198 -0.834 1.520 -ATOM 3584 H1 TIP3 3776 -0.629 18.319 -17.904 0.417 1.000 -ATOM 3585 H2 TIP3 3776 0.139 17.007 -18.337 0.417 1.000 -ATOM 3586 OH2 TIP3 3779 6.637 16.087 -18.996 -0.834 1.520 -ATOM 3587 H1 TIP3 3779 7.498 16.252 -19.407 0.417 1.000 -ATOM 3588 H2 TIP3 3779 6.017 16.028 -19.745 0.417 1.000 -ATOM 3589 OH2 TIP3 3782 8.481 11.362 3.178 -0.834 1.520 -ATOM 3590 H1 TIP3 3782 9.115 11.898 2.672 0.417 1.000 -ATOM 3591 H2 TIP3 3782 8.051 12.015 3.779 0.417 1.000 -ATOM 3592 OH2 TIP3 3786 18.592 18.819 -4.876 -0.834 1.520 -ATOM 3593 H1 TIP3 3786 18.085 18.482 -4.102 0.417 1.000 -ATOM 3594 H2 TIP3 3786 19.458 18.516 -4.519 0.417 1.000 -ATOM 3595 OH2 TIP3 3787 1.899 1.156 4.376 -0.834 1.520 -ATOM 3596 H1 TIP3 3787 2.735 0.786 4.748 0.417 1.000 -ATOM 3597 H2 TIP3 3787 2.058 2.130 4.308 0.417 1.000 -ATOM 3598 OH2 TIP3 3788 -4.347 11.558 10.925 -0.834 1.520 -ATOM 3599 H1 TIP3 3788 -4.052 11.753 11.873 0.417 1.000 -ATOM 3600 H2 TIP3 3788 -5.113 12.079 10.837 0.417 1.000 -ATOM 3601 OH2 TIP3 3789 8.509 15.644 13.685 -0.834 1.520 -ATOM 3602 H1 TIP3 3789 9.225 15.098 13.985 0.417 1.000 -ATOM 3603 H2 TIP3 3789 7.910 15.868 14.500 0.417 1.000 -ATOM 3604 OH2 TIP3 3790 1.536 16.060 2.084 -0.834 1.520 -ATOM 3605 H1 TIP3 3790 0.892 16.677 2.538 0.417 1.000 -ATOM 3606 H2 TIP3 3790 1.559 15.350 2.741 0.417 1.000 -ATOM 3607 OH2 TIP3 3792 6.590 9.316 5.618 -0.834 1.520 -ATOM 3608 H1 TIP3 3792 6.197 10.093 6.064 0.417 1.000 -ATOM 3609 H2 TIP3 3792 5.854 8.815 5.258 0.417 1.000 -ATOM 3610 OH2 TIP3 3793 13.738 15.564 15.836 -0.834 1.520 -ATOM 3611 H1 TIP3 3793 13.875 15.700 16.760 0.417 1.000 -ATOM 3612 H2 TIP3 3793 14.687 15.318 15.462 0.417 1.000 -ATOM 3613 OH2 TIP3 3796 0.923 17.067 12.534 -0.834 1.520 -ATOM 3614 H1 TIP3 3796 0.951 16.521 11.751 0.417 1.000 -ATOM 3615 H2 TIP3 3796 0.543 16.553 13.240 0.417 1.000 -ATOM 3616 OH2 TIP3 3801 14.969 16.609 -19.485 -0.834 1.520 -ATOM 3617 H1 TIP3 3801 14.822 17.562 -19.310 0.417 1.000 -ATOM 3618 H2 TIP3 3801 14.532 16.181 -18.769 0.417 1.000 -ATOM 3619 OH2 TIP3 3814 3.934 16.036 13.741 -0.834 1.520 -ATOM 3620 H1 TIP3 3814 3.190 15.530 13.372 0.417 1.000 -ATOM 3621 H2 TIP3 3814 3.532 16.661 14.417 0.417 1.000 -ATOM 3622 OH2 TIP3 3821 13.010 6.672 -12.664 -0.834 1.520 -ATOM 3623 H1 TIP3 3821 12.982 7.638 -12.578 0.417 1.000 -ATOM 3624 H2 TIP3 3821 12.347 6.374 -12.045 0.417 1.000 -ATOM 3625 OH2 TIP3 3829 12.231 10.996 -6.292 -0.834 1.520 -ATOM 3626 H1 TIP3 3829 11.331 11.344 -6.120 0.417 1.000 -ATOM 3627 H2 TIP3 3829 12.051 10.275 -6.944 0.417 1.000 -ATOM 3628 OH2 TIP3 3833 6.348 18.122 9.123 -0.834 1.520 -ATOM 3629 H1 TIP3 3833 7.110 17.449 9.118 0.417 1.000 -ATOM 3630 H2 TIP3 3833 5.977 17.917 9.990 0.417 1.000 -ATOM 3631 OH2 TIP3 3834 2.904 3.633 14.112 -0.834 1.520 -ATOM 3632 H1 TIP3 3834 1.945 3.842 13.956 0.417 1.000 -ATOM 3633 H2 TIP3 3834 3.299 3.749 13.233 0.417 1.000 -ATOM 3634 OH2 TIP3 3872 4.418 13.938 10.661 -0.834 1.520 -ATOM 3635 H1 TIP3 3872 3.952 14.846 10.630 0.417 1.000 -ATOM 3636 H2 TIP3 3872 3.716 13.327 11.027 0.417 1.000 -ATOM 3637 OH2 TIP3 3892 11.246 13.527 -2.479 -0.834 1.520 -ATOM 3638 H1 TIP3 3892 11.898 12.901 -2.121 0.417 1.000 -ATOM 3639 H2 TIP3 3892 10.392 13.003 -2.459 0.417 1.000 -ATOM 3640 OH2 TIP3 3924 2.436 -6.837 13.707 -0.834 1.520 -ATOM 3641 H1 TIP3 3924 2.654 -7.154 12.787 0.417 1.000 -ATOM 3642 H2 TIP3 3924 1.437 -6.628 13.598 0.417 1.000 -ATOM 3643 OH2 TIP3 3925 2.567 -19.240 -4.911 -0.834 1.520 -ATOM 3644 H1 TIP3 3925 3.323 -19.105 -4.332 0.417 1.000 -ATOM 3645 H2 TIP3 3925 2.175 -18.298 -4.915 0.417 1.000 -ATOM 3646 OH2 TIP3 3931 10.847 -10.906 -13.051 -0.834 1.520 -ATOM 3647 H1 TIP3 3931 10.536 -10.501 -12.215 0.417 1.000 -ATOM 3648 H2 TIP3 3931 11.760 -11.088 -12.731 0.417 1.000 -ATOM 3649 OH2 TIP3 3933 0.520 -14.297 -8.575 -0.834 1.520 -ATOM 3650 H1 TIP3 3933 -0.250 -14.841 -8.280 0.417 1.000 -ATOM 3651 H2 TIP3 3933 0.178 -13.791 -9.402 0.417 1.000 -ATOM 3652 OH2 TIP3 3935 -2.154 -10.960 5.906 -0.834 1.520 -ATOM 3653 H1 TIP3 3935 -2.245 -11.755 5.370 0.417 1.000 -ATOM 3654 H2 TIP3 3935 -1.296 -10.640 5.724 0.417 1.000 -ATOM 3655 OH2 TIP3 3938 8.079 -10.311 19.229 -0.834 1.520 -ATOM 3656 H1 TIP3 3938 7.342 -9.777 18.926 0.417 1.000 -ATOM 3657 H2 TIP3 3938 8.575 -10.398 18.386 0.417 1.000 -ATOM 3658 OH2 TIP3 3951 0.969 -18.875 -10.175 -0.834 1.520 -ATOM 3659 H1 TIP3 3951 1.514 -18.160 -9.758 0.417 1.000 -ATOM 3660 H2 TIP3 3951 0.448 -18.367 -10.822 0.417 1.000 -ATOM 3661 OH2 TIP3 3957 17.252 -4.235 12.061 -0.834 1.520 -ATOM 3662 H1 TIP3 3957 17.786 -5.004 11.783 0.417 1.000 -ATOM 3663 H2 TIP3 3957 17.778 -3.936 12.830 0.417 1.000 -ATOM 3664 OH2 TIP3 3974 8.638 -14.902 -0.868 -0.834 1.520 -ATOM 3665 H1 TIP3 3974 8.603 -15.162 -1.789 0.417 1.000 -ATOM 3666 H2 TIP3 3974 7.854 -14.352 -0.783 0.417 1.000 -ATOM 3667 OH2 TIP3 3975 11.657 -12.587 -10.331 -0.834 1.520 -ATOM 3668 H1 TIP3 3975 11.102 -13.101 -10.955 0.417 1.000 -ATOM 3669 H2 TIP3 3975 11.276 -11.690 -10.218 0.417 1.000 -ATOM 3670 OH2 TIP3 3976 -2.483 -8.142 -6.990 -0.834 1.520 -ATOM 3671 H1 TIP3 3976 -2.360 -8.227 -7.961 0.417 1.000 -ATOM 3672 H2 TIP3 3976 -2.711 -7.219 -6.870 0.417 1.000 -ATOM 3673 OH2 TIP3 3977 -11.100 -12.690 -8.710 -0.834 1.520 -ATOM 3674 H1 TIP3 3977 -11.261 -13.294 -7.922 0.417 1.000 -ATOM 3675 H2 TIP3 3977 -10.445 -11.997 -8.469 0.417 1.000 -ATOM 3676 OH2 TIP3 3982 6.085 -17.763 16.922 -0.834 1.520 -ATOM 3677 H1 TIP3 3982 6.912 -17.705 16.416 0.417 1.000 -ATOM 3678 H2 TIP3 3982 5.808 -18.656 16.647 0.417 1.000 -ATOM 3679 OH2 TIP3 3986 7.820 -17.527 -19.066 -0.834 1.520 -ATOM 3680 H1 TIP3 3986 8.525 -16.831 -19.113 0.417 1.000 -ATOM 3681 H2 TIP3 3986 7.193 -17.001 -18.501 0.417 1.000 -ATOM 3682 OH2 TIP3 3991 -2.790 -16.954 -3.228 -0.834 1.520 -ATOM 3683 H1 TIP3 3991 -3.225 -16.377 -3.908 0.417 1.000 -ATOM 3684 H2 TIP3 3991 -2.319 -16.271 -2.745 0.417 1.000 -ATOM 3685 OH2 TIP3 3992 8.814 -13.094 -12.626 -0.834 1.520 -ATOM 3686 H1 TIP3 3992 8.585 -12.163 -12.441 0.417 1.000 -ATOM 3687 H2 TIP3 3992 7.928 -13.425 -12.813 0.417 1.000 -ATOM 3688 OH2 TIP3 3995 0.736 -3.581 -1.447 -0.834 1.520 -ATOM 3689 H1 TIP3 3995 0.764 -3.671 -2.443 0.417 1.000 -ATOM 3690 H2 TIP3 3995 1.696 -3.517 -1.213 0.417 1.000 -ATOM 3691 OH2 TIP3 3996 -0.049 -13.567 -15.694 -0.834 1.520 -ATOM 3692 H1 TIP3 3996 -0.404 -12.598 -15.541 0.417 1.000 -ATOM 3693 H2 TIP3 3996 -0.446 -14.069 -14.972 0.417 1.000 -ATOM 3694 OH2 TIP3 3997 12.718 -14.652 7.053 -0.834 1.520 -ATOM 3695 H1 TIP3 3997 13.603 -14.981 6.921 0.417 1.000 -ATOM 3696 H2 TIP3 3997 12.922 -13.638 6.971 0.417 1.000 -ATOM 3697 OH2 TIP3 3998 15.440 -7.081 9.100 -0.834 1.520 -ATOM 3698 H1 TIP3 3998 14.838 -7.743 8.881 0.417 1.000 -ATOM 3699 H2 TIP3 3998 14.992 -6.594 9.885 0.417 1.000 -ATOM 3700 OH2 TIP3 3999 9.861 -16.864 18.399 -0.834 1.520 -ATOM 3701 H1 TIP3 3999 9.833 -17.832 18.395 0.417 1.000 -ATOM 3702 H2 TIP3 3999 10.588 -16.686 17.801 0.417 1.000 -ATOM 3703 OH2 TIP3 4000 13.135 -3.562 9.141 -0.834 1.520 -ATOM 3704 H1 TIP3 4000 13.619 -4.080 9.751 0.417 1.000 -ATOM 3705 H2 TIP3 4000 12.247 -3.589 9.491 0.417 1.000 -ATOM 3706 OH2 TIP3 4002 16.528 -13.601 18.790 -0.834 1.520 -ATOM 3707 H1 TIP3 4002 16.534 -14.037 17.936 0.417 1.000 -ATOM 3708 H2 TIP3 4002 17.180 -12.868 18.599 0.417 1.000 -ATOM 3709 OH2 TIP3 4008 12.422 -6.895 15.788 -0.834 1.520 -ATOM 3710 H1 TIP3 4008 11.988 -7.688 16.179 0.417 1.000 -ATOM 3711 H2 TIP3 4008 13.416 -7.040 15.895 0.417 1.000 -ATOM 3712 OH2 TIP3 4009 1.334 -12.778 -3.130 -0.834 1.520 -ATOM 3713 H1 TIP3 4009 1.033 -13.557 -2.561 0.417 1.000 -ATOM 3714 H2 TIP3 4009 2.137 -12.517 -2.756 0.417 1.000 -ATOM 3715 OH2 TIP3 4010 13.590 4.057 -9.656 -0.834 1.520 -ATOM 3716 H1 TIP3 4010 14.102 3.354 -10.072 0.417 1.000 -ATOM 3717 H2 TIP3 4010 12.721 3.865 -10.077 0.417 1.000 -ATOM 3718 OH2 TIP3 4012 -10.505 -5.584 -18.021 -0.834 1.520 -ATOM 3719 H1 TIP3 4012 -9.646 -5.797 -18.452 0.417 1.000 -ATOM 3720 H2 TIP3 4012 -10.835 -6.434 -17.901 0.417 1.000 -ATOM 3721 OH2 TIP3 4014 10.874 -18.934 -9.095 -0.834 1.520 -ATOM 3722 H1 TIP3 4014 10.395 -19.783 -8.954 0.417 1.000 -ATOM 3723 H2 TIP3 4014 10.290 -18.455 -9.664 0.417 1.000 -ATOM 3724 OH2 TIP3 4015 5.364 -18.735 -12.866 -0.834 1.520 -ATOM 3725 H1 TIP3 4015 4.665 -19.036 -12.210 0.417 1.000 -ATOM 3726 H2 TIP3 4015 5.826 -19.522 -12.955 0.417 1.000 -ATOM 3727 OH2 TIP3 4016 8.064 -3.052 -11.241 -0.834 1.520 -ATOM 3728 H1 TIP3 4016 8.687 -3.318 -10.573 0.417 1.000 -ATOM 3729 H2 TIP3 4016 8.126 -3.743 -11.911 0.417 1.000 -ATOM 3730 OH2 TIP3 4017 3.065 -15.697 -6.374 -0.834 1.520 -ATOM 3731 H1 TIP3 4017 3.694 -14.912 -6.437 0.417 1.000 -ATOM 3732 H2 TIP3 4017 3.274 -16.119 -7.215 0.417 1.000 -ATOM 3733 OH2 TIP3 4018 10.995 -8.500 3.703 -0.834 1.520 -ATOM 3734 H1 TIP3 4018 10.653 -9.321 4.022 0.417 1.000 -ATOM 3735 H2 TIP3 4018 11.942 -8.744 3.491 0.417 1.000 -ATOM 3736 OH2 TIP3 4019 1.576 -8.932 15.249 -0.834 1.520 -ATOM 3737 H1 TIP3 4019 1.856 -8.125 14.811 0.417 1.000 -ATOM 3738 H2 TIP3 4019 0.857 -8.663 15.816 0.417 1.000 -ATOM 3739 OH2 TIP3 4020 -1.606 2.097 15.921 -0.834 1.520 -ATOM 3740 H1 TIP3 4020 -1.872 1.729 16.726 0.417 1.000 -ATOM 3741 H2 TIP3 4020 -0.753 2.630 16.127 0.417 1.000 -ATOM 3742 OH2 TIP3 4021 10.622 -13.934 0.635 -0.834 1.520 -ATOM 3743 H1 TIP3 4021 10.281 -13.753 1.496 0.417 1.000 -ATOM 3744 H2 TIP3 4021 9.861 -14.147 0.054 0.417 1.000 -ATOM 3745 OH2 TIP3 4022 14.702 -11.815 -0.918 -0.834 1.520 -ATOM 3746 H1 TIP3 4022 14.386 -12.520 -0.405 0.417 1.000 -ATOM 3747 H2 TIP3 4022 14.028 -11.782 -1.689 0.417 1.000 -ATOM 3748 OH2 TIP3 4033 -4.631 16.264 -14.389 -0.834 1.520 -ATOM 3749 H1 TIP3 4033 -4.414 16.956 -13.783 0.417 1.000 -ATOM 3750 H2 TIP3 4033 -4.630 15.505 -13.834 0.417 1.000 -ATOM 3751 OH2 TIP3 4035 6.318 -13.975 -12.652 -0.834 1.520 -ATOM 3752 H1 TIP3 4035 6.091 -14.638 -12.062 0.417 1.000 -ATOM 3753 H2 TIP3 4035 5.538 -13.834 -13.227 0.417 1.000 -ATOM 3754 OH2 TIP3 4036 -6.395 -14.647 -0.720 -0.834 1.520 -ATOM 3755 H1 TIP3 4036 -6.342 -15.473 -1.188 0.417 1.000 -ATOM 3756 H2 TIP3 4036 -7.312 -14.633 -0.484 0.417 1.000 -ATOM 3757 OH2 TIP3 4037 -0.057 -11.459 -18.909 -0.834 1.520 -ATOM 3758 H1 TIP3 4037 -0.292 -12.409 -18.731 0.417 1.000 -ATOM 3759 H2 TIP3 4037 -0.863 -10.967 -18.795 0.417 1.000 -ATOM 3760 OH2 TIP3 4038 3.521 11.765 8.560 -0.834 1.520 -ATOM 3761 H1 TIP3 4038 3.477 11.852 9.555 0.417 1.000 -ATOM 3762 H2 TIP3 4038 2.667 12.153 8.282 0.417 1.000 -ATOM 3763 OH2 TIP3 4039 -0.516 -14.181 1.991 -0.834 1.520 -ATOM 3764 H1 TIP3 4039 -0.814 -13.826 2.828 0.417 1.000 -ATOM 3765 H2 TIP3 4039 -0.266 -13.368 1.445 0.417 1.000 -ATOM 3766 OH2 TIP3 4041 5.896 -14.636 8.742 -0.834 1.520 -ATOM 3767 H1 TIP3 4041 6.865 -14.591 8.639 0.417 1.000 -ATOM 3768 H2 TIP3 4041 5.662 -13.782 9.195 0.417 1.000 -ATOM 3769 OH2 TIP3 4046 5.468 -9.041 18.390 -0.834 1.520 -ATOM 3770 H1 TIP3 4046 5.292 -8.146 18.137 0.417 1.000 -ATOM 3771 H2 TIP3 4046 4.747 -9.595 18.029 0.417 1.000 -ATOM 3772 OH2 TIP3 4047 4.494 -5.204 14.953 -0.834 1.520 -ATOM 3773 H1 TIP3 4047 5.297 -5.596 14.521 0.417 1.000 -ATOM 3774 H2 TIP3 4047 3.764 -5.588 14.441 0.417 1.000 -ATOM 3775 OH2 TIP3 4053 17.225 -10.527 -13.145 -0.834 1.520 -ATOM 3776 H1 TIP3 4053 16.967 -11.229 -13.761 0.417 1.000 -ATOM 3777 H2 TIP3 4053 16.399 -10.177 -12.786 0.417 1.000 -ATOM 3778 OH2 TIP3 4054 12.874 5.033 -3.059 -0.834 1.520 -ATOM 3779 H1 TIP3 4054 13.783 4.696 -3.140 0.417 1.000 -ATOM 3780 H2 TIP3 4054 12.868 5.147 -2.082 0.417 1.000 -ATOM 3781 OH2 TIP3 4055 7.765 0.981 -19.146 -0.834 1.520 -ATOM 3782 H1 TIP3 4055 7.191 0.959 -19.961 0.417 1.000 -ATOM 3783 H2 TIP3 4055 7.432 1.832 -18.752 0.417 1.000 -ATOM 3784 OH2 TIP3 4056 10.722 -0.680 -4.476 -0.834 1.520 -ATOM 3785 H1 TIP3 4056 11.315 -0.019 -4.094 0.417 1.000 -ATOM 3786 H2 TIP3 4056 11.095 -0.738 -5.376 0.417 1.000 -ATOM 3787 OH2 TIP3 4057 14.612 2.300 -12.985 -0.834 1.520 -ATOM 3788 H1 TIP3 4057 14.494 1.817 -13.831 0.417 1.000 -ATOM 3789 H2 TIP3 4057 14.500 3.235 -13.236 0.417 1.000 -ATOM 3790 OH2 TIP3 4059 3.112 6.630 -4.085 -0.834 1.520 -ATOM 3791 H1 TIP3 4059 2.633 6.888 -4.904 0.417 1.000 -ATOM 3792 H2 TIP3 4059 3.826 6.040 -4.348 0.417 1.000 -ATOM 3793 OH2 TIP3 4060 15.161 -2.571 -6.877 -0.834 1.520 -ATOM 3794 H1 TIP3 4060 15.258 -3.527 -6.987 0.417 1.000 -ATOM 3795 H2 TIP3 4060 15.162 -2.327 -7.852 0.417 1.000 -ATOM 3796 OH2 TIP3 4062 8.951 -18.557 8.480 -0.834 1.520 -ATOM 3797 H1 TIP3 4062 9.851 -18.163 8.465 0.417 1.000 -ATOM 3798 H2 TIP3 4062 8.799 -18.820 7.540 0.417 1.000 -ATOM 3799 OH2 TIP3 4064 18.327 5.035 -1.685 -0.834 1.520 -ATOM 3800 H1 TIP3 4064 18.846 5.491 -1.050 0.417 1.000 -ATOM 3801 H2 TIP3 4064 18.154 4.160 -1.397 0.417 1.000 -ATOM 3802 OH2 TIP3 4065 -0.212 -6.172 13.553 -0.834 1.520 -ATOM 3803 H1 TIP3 4065 -0.433 -5.383 12.946 0.417 1.000 -ATOM 3804 H2 TIP3 4065 -0.946 -6.745 13.502 0.417 1.000 -ATOM 3805 OH2 TIP3 4066 5.637 8.363 17.002 -0.834 1.520 -ATOM 3806 H1 TIP3 4066 5.026 8.071 17.718 0.417 1.000 -ATOM 3807 H2 TIP3 4066 6.414 8.745 17.464 0.417 1.000 -ATOM 3808 OH2 TIP3 4074 -12.666 -2.027 -12.910 -0.834 1.520 -ATOM 3809 H1 TIP3 4074 -12.429 -1.610 -13.800 0.417 1.000 -ATOM 3810 H2 TIP3 4074 -13.467 -2.556 -13.115 0.417 1.000 -ATOM 3811 OH2 TIP3 4076 6.108 -3.939 -8.361 -0.834 1.520 -ATOM 3812 H1 TIP3 4076 6.183 -3.416 -9.193 0.417 1.000 -ATOM 3813 H2 TIP3 4076 6.317 -3.316 -7.699 0.417 1.000 -ATOM 3814 OH2 TIP3 4077 -6.088 -8.960 -2.617 -0.834 1.520 -ATOM 3815 H1 TIP3 4077 -6.087 -9.068 -3.577 0.417 1.000 -ATOM 3816 H2 TIP3 4077 -6.388 -8.065 -2.482 0.417 1.000 -ATOM 3817 OH2 TIP3 4079 -3.025 -2.427 5.147 -0.834 1.520 -ATOM 3818 H1 TIP3 4079 -3.222 -1.571 5.632 0.417 1.000 -ATOM 3819 H2 TIP3 4079 -3.822 -2.680 4.650 0.417 1.000 -ATOM 3820 OH2 TIP3 4080 8.662 7.102 13.839 -0.834 1.520 -ATOM 3821 H1 TIP3 4080 8.504 8.001 13.610 0.417 1.000 -ATOM 3822 H2 TIP3 4080 9.019 6.810 13.024 0.417 1.000 -ATOM 3823 OH2 TIP3 4081 1.807 -0.308 1.956 -0.834 1.520 -ATOM 3824 H1 TIP3 4081 1.892 -1.071 2.570 0.417 1.000 -ATOM 3825 H2 TIP3 4081 1.560 0.390 2.634 0.417 1.000 -ATOM 3826 OH2 TIP3 4082 6.226 -10.810 14.878 -0.834 1.520 -ATOM 3827 H1 TIP3 4082 6.258 -11.728 14.962 0.417 1.000 -ATOM 3828 H2 TIP3 4082 7.135 -10.576 14.667 0.417 1.000 -ATOM 3829 OH2 TIP3 4083 4.469 5.918 -1.441 -0.834 1.520 -ATOM 3830 H1 TIP3 4083 4.369 6.179 -2.406 0.417 1.000 -ATOM 3831 H2 TIP3 4083 5.495 6.011 -1.365 0.417 1.000 -ATOM 3832 OH2 TIP3 4084 11.780 -11.246 5.299 -0.834 1.520 -ATOM 3833 H1 TIP3 4084 12.565 -11.144 4.723 0.417 1.000 -ATOM 3834 H2 TIP3 4084 12.088 -11.048 6.204 0.417 1.000 -ATOM 3835 OH2 TIP3 4085 2.776 3.126 18.490 -0.834 1.520 -ATOM 3836 H1 TIP3 4085 2.530 3.297 19.414 0.417 1.000 -ATOM 3837 H2 TIP3 4085 3.724 2.831 18.576 0.417 1.000 -ATOM 3838 OH2 TIP3 4093 12.300 10.502 -14.924 -0.834 1.520 -ATOM 3839 H1 TIP3 4093 12.005 10.159 -15.743 0.417 1.000 -ATOM 3840 H2 TIP3 4093 11.914 11.375 -14.817 0.417 1.000 -ATOM 3841 OH2 TIP3 4096 13.515 -0.385 5.948 -0.834 1.520 -ATOM 3842 H1 TIP3 4096 13.676 -0.628 6.851 0.417 1.000 -ATOM 3843 H2 TIP3 4096 14.031 0.424 5.802 0.417 1.000 -ATOM 3844 OH2 TIP3 4097 1.975 10.136 -16.634 -0.834 1.520 -ATOM 3845 H1 TIP3 4097 1.739 9.870 -15.758 0.417 1.000 -ATOM 3846 H2 TIP3 4097 1.767 11.134 -16.723 0.417 1.000 -ATOM 3847 OH2 TIP3 4098 13.985 -7.680 -11.033 -0.834 1.520 -ATOM 3848 H1 TIP3 4098 13.589 -7.939 -10.182 0.417 1.000 -ATOM 3849 H2 TIP3 4098 14.317 -8.492 -11.470 0.417 1.000 -ATOM 3850 OH2 TIP3 4099 10.533 -10.235 -1.252 -0.834 1.520 -ATOM 3851 H1 TIP3 4099 10.567 -9.300 -1.058 0.417 1.000 -ATOM 3852 H2 TIP3 4099 9.591 -10.461 -1.351 0.417 1.000 -ATOM 3853 OH2 TIP3 4102 -4.275 4.795 7.901 -0.834 1.520 -ATOM 3854 H1 TIP3 4102 -4.818 4.055 7.722 0.417 1.000 -ATOM 3855 H2 TIP3 4102 -3.748 4.835 7.062 0.417 1.000 -ATOM 3856 OH2 TIP3 4103 12.539 -3.322 12.531 -0.834 1.520 -ATOM 3857 H1 TIP3 4103 11.809 -2.734 12.809 0.417 1.000 -ATOM 3858 H2 TIP3 4103 13.199 -3.256 13.220 0.417 1.000 -ATOM 3859 OH2 TIP3 4104 -2.720 -5.813 2.328 -0.834 1.520 -ATOM 3860 H1 TIP3 4104 -3.074 -6.151 1.493 0.417 1.000 -ATOM 3861 H2 TIP3 4104 -1.809 -6.180 2.308 0.417 1.000 -ATOM 3862 OH2 TIP3 4105 7.964 9.704 18.059 -0.834 1.520 -ATOM 3863 H1 TIP3 4105 8.117 10.373 18.705 0.417 1.000 -ATOM 3864 H2 TIP3 4105 8.792 9.595 17.687 0.417 1.000 -ATOM 3865 OH2 TIP3 4111 5.610 -0.911 15.594 -0.834 1.520 -ATOM 3866 H1 TIP3 4111 4.892 -0.302 15.373 0.417 1.000 -ATOM 3867 H2 TIP3 4111 5.107 -1.675 15.959 0.417 1.000 -ATOM 3868 OH2 TIP3 4116 8.314 11.034 -13.488 -0.834 1.520 -ATOM 3869 H1 TIP3 4116 8.977 10.813 -12.830 0.417 1.000 -ATOM 3870 H2 TIP3 4116 7.711 10.233 -13.413 0.417 1.000 -ATOM 3871 OH2 TIP3 4120 18.872 10.165 -7.349 -0.834 1.520 -ATOM 3872 H1 TIP3 4120 19.333 9.322 -7.181 0.417 1.000 -ATOM 3873 H2 TIP3 4120 19.365 10.598 -8.107 0.417 1.000 -ATOM 3874 OH2 TIP3 4121 8.897 -1.486 -8.849 -0.834 1.520 -ATOM 3875 H1 TIP3 4121 8.880 -1.932 -8.002 0.417 1.000 -ATOM 3876 H2 TIP3 4121 7.887 -1.323 -8.982 0.417 1.000 -ATOM 3877 OH2 TIP3 4122 12.032 2.534 9.832 -0.834 1.520 -ATOM 3878 H1 TIP3 4122 12.926 2.262 9.985 0.417 1.000 -ATOM 3879 H2 TIP3 4122 11.952 2.477 8.841 0.417 1.000 -ATOM 3880 OH2 TIP3 4123 18.641 8.747 -3.397 -0.834 1.520 -ATOM 3881 H1 TIP3 4123 17.669 8.714 -3.558 0.417 1.000 -ATOM 3882 H2 TIP3 4123 18.986 8.413 -4.231 0.417 1.000 -ATOM 3883 OH2 TIP3 4127 8.501 2.159 11.404 -0.834 1.520 -ATOM 3884 H1 TIP3 4127 8.956 2.452 12.204 0.417 1.000 -ATOM 3885 H2 TIP3 4127 7.774 2.841 11.233 0.417 1.000 -ATOM 3886 OH2 TIP3 4138 8.044 -1.600 -17.478 -0.834 1.520 -ATOM 3887 H1 TIP3 4138 7.927 -0.870 -18.225 0.417 1.000 -ATOM 3888 H2 TIP3 4138 7.159 -1.823 -17.185 0.417 1.000 -ATOM 3889 OH2 TIP3 4139 6.481 7.553 -10.971 -0.834 1.520 -ATOM 3890 H1 TIP3 4139 6.004 7.381 -10.096 0.417 1.000 -ATOM 3891 H2 TIP3 4139 5.929 7.078 -11.618 0.417 1.000 -ATOM 3892 OH2 TIP3 4140 8.903 12.212 -1.010 -0.834 1.520 -ATOM 3893 H1 TIP3 4140 8.815 11.966 -1.916 0.417 1.000 -ATOM 3894 H2 TIP3 4140 8.006 12.124 -0.704 0.417 1.000 -ATOM 3895 OH2 TIP3 4141 1.720 0.988 -11.947 -0.834 1.520 -ATOM 3896 H1 TIP3 4141 2.147 0.491 -11.220 0.417 1.000 -ATOM 3897 H2 TIP3 4141 2.486 1.373 -12.378 0.417 1.000 -ATOM 3898 OH2 TIP3 4142 19.270 5.575 -17.443 -0.834 1.520 -ATOM 3899 H1 TIP3 4142 18.738 4.855 -17.786 0.417 1.000 -ATOM 3900 H2 TIP3 4142 18.629 6.323 -17.615 0.417 1.000 -ATOM 3901 OH2 TIP3 4143 6.601 1.149 -7.345 -0.834 1.520 -ATOM 3902 H1 TIP3 4143 7.129 1.882 -7.020 0.417 1.000 -ATOM 3903 H2 TIP3 4143 6.906 0.393 -6.862 0.417 1.000 -ATOM 3904 OH2 TIP3 4144 11.096 -13.224 3.657 -0.834 1.520 -ATOM 3905 H1 TIP3 4144 11.988 -13.476 3.265 0.417 1.000 -ATOM 3906 H2 TIP3 4144 11.323 -12.607 4.336 0.417 1.000 -ATOM 3907 OH2 TIP3 4146 12.044 8.291 6.730 -0.834 1.520 -ATOM 3908 H1 TIP3 4146 12.897 8.113 6.127 0.417 1.000 -ATOM 3909 H2 TIP3 4146 11.822 9.248 6.625 0.417 1.000 -ATOM 3910 OH2 TIP3 4147 4.689 17.948 7.004 -0.834 1.520 -ATOM 3911 H1 TIP3 4147 5.333 17.874 7.739 0.417 1.000 -ATOM 3912 H2 TIP3 4147 3.892 17.654 7.383 0.417 1.000 -ATOM 3913 OH2 TIP3 4148 10.853 11.643 16.882 -0.834 1.520 -ATOM 3914 H1 TIP3 4148 10.367 12.511 16.824 0.417 1.000 -ATOM 3915 H2 TIP3 4148 11.707 11.950 17.265 0.417 1.000 -ATOM 3916 OH2 TIP3 4150 3.171 11.918 15.414 -0.834 1.520 -ATOM 3917 H1 TIP3 4150 3.217 11.076 14.972 0.417 1.000 -ATOM 3918 H2 TIP3 4150 4.123 12.244 15.353 0.417 1.000 -ATOM 3919 OH2 TIP3 4151 14.251 2.876 14.281 -0.834 1.520 -ATOM 3920 H1 TIP3 4151 14.147 1.926 14.259 0.417 1.000 -ATOM 3921 H2 TIP3 4151 13.377 3.250 14.123 0.417 1.000 -ATOM 3922 OH2 TIP3 4157 7.107 8.478 -14.016 -0.834 1.520 -ATOM 3923 H1 TIP3 4157 7.630 7.674 -14.261 0.417 1.000 -ATOM 3924 H2 TIP3 4157 6.224 8.217 -14.144 0.417 1.000 -ATOM 3925 OH2 TIP3 4158 4.943 19.812 -16.720 -0.834 1.520 -ATOM 3926 H1 TIP3 4158 4.840 18.876 -16.851 0.417 1.000 -ATOM 3927 H2 TIP3 4158 5.901 19.929 -16.832 0.417 1.000 -ATOM 3928 OH2 TIP3 4161 11.156 4.433 -18.883 -0.834 1.520 -ATOM 3929 H1 TIP3 4161 10.846 4.181 -17.998 0.417 1.000 -ATOM 3930 H2 TIP3 4161 11.896 3.831 -18.923 0.417 1.000 -ATOM 3931 OH2 TIP3 4162 0.504 11.104 3.859 -0.834 1.520 -ATOM 3932 H1 TIP3 4162 0.195 11.056 4.816 0.417 1.000 -ATOM 3933 H2 TIP3 4162 0.136 10.246 3.526 0.417 1.000 -ATOM 3934 OH2 TIP3 4163 0.735 -0.181 7.128 -0.834 1.520 -ATOM 3935 H1 TIP3 4163 -0.115 -0.007 7.429 0.417 1.000 -ATOM 3936 H2 TIP3 4163 0.766 0.363 6.302 0.417 1.000 -ATOM 3937 OH2 TIP3 4165 9.874 8.123 -1.393 -0.834 1.520 -ATOM 3938 H1 TIP3 4165 9.015 8.213 -0.949 0.417 1.000 -ATOM 3939 H2 TIP3 4165 10.498 8.465 -0.767 0.417 1.000 -ATOM 3940 OH2 TIP3 4166 15.533 4.218 16.427 -0.834 1.520 -ATOM 3941 H1 TIP3 4166 14.911 4.824 15.962 0.417 1.000 -ATOM 3942 H2 TIP3 4166 15.288 3.372 16.001 0.417 1.000 -ATOM 3943 OH2 TIP3 4167 5.634 13.114 5.143 -0.834 1.520 -ATOM 3944 H1 TIP3 4167 6.549 13.396 5.248 0.417 1.000 -ATOM 3945 H2 TIP3 4167 5.585 12.459 5.932 0.417 1.000 -ATOM 3946 OH2 TIP3 4168 15.325 19.232 14.039 -0.834 1.520 -ATOM 3947 H1 TIP3 4168 16.210 18.867 14.030 0.417 1.000 -ATOM 3948 H2 TIP3 4168 15.402 19.820 14.803 0.417 1.000 -ATOM 3949 OH2 TIP3 4179 6.844 3.498 -18.022 -0.834 1.520 -ATOM 3950 H1 TIP3 4179 6.643 4.419 -17.863 0.417 1.000 -ATOM 3951 H2 TIP3 4179 6.537 3.018 -17.213 0.417 1.000 -ATOM 3952 OH2 TIP3 4181 5.540 10.998 -16.668 -0.834 1.520 -ATOM 3953 H1 TIP3 4181 5.672 11.950 -16.477 0.417 1.000 -ATOM 3954 H2 TIP3 4181 6.411 10.658 -16.461 0.417 1.000 -ATOM 3955 OH2 TIP3 4184 3.350 5.426 -17.351 -0.834 1.520 -ATOM 3956 H1 TIP3 4184 4.136 5.667 -17.925 0.417 1.000 -ATOM 3957 H2 TIP3 4184 2.685 6.158 -17.650 0.417 1.000 -ATOM 3958 OH2 TIP3 4185 1.361 6.625 11.578 -0.834 1.520 -ATOM 3959 H1 TIP3 4185 1.071 5.884 12.149 0.417 1.000 -ATOM 3960 H2 TIP3 4185 0.794 7.382 11.851 0.417 1.000 -ATOM 3961 OH2 TIP3 4186 8.010 3.657 -6.705 -0.834 1.520 -ATOM 3962 H1 TIP3 4186 8.334 3.472 -5.800 0.417 1.000 -ATOM 3963 H2 TIP3 4186 8.782 3.742 -7.168 0.417 1.000 -ATOM 3964 OH2 TIP3 4188 0.036 13.982 19.637 -0.834 1.520 -ATOM 3965 H1 TIP3 4188 0.203 14.913 19.756 0.417 1.000 -ATOM 3966 H2 TIP3 4188 0.714 13.737 18.982 0.417 1.000 -ATOM 3967 OH2 TIP3 4191 17.828 8.861 1.960 -0.834 1.520 -ATOM 3968 H1 TIP3 4191 17.815 9.267 1.103 0.417 1.000 -ATOM 3969 H2 TIP3 4191 17.384 9.537 2.444 0.417 1.000 -ATOM 3970 OH2 TIP3 4198 14.961 18.217 -15.090 -0.834 1.520 -ATOM 3971 H1 TIP3 4198 15.628 18.919 -15.326 0.417 1.000 -ATOM 3972 H2 TIP3 4198 14.123 18.595 -15.436 0.417 1.000 -ATOM 3973 OH2 TIP3 4201 16.903 3.552 -16.985 -0.834 1.520 -ATOM 3974 H1 TIP3 4201 17.054 3.149 -16.144 0.417 1.000 -ATOM 3975 H2 TIP3 4201 16.843 4.518 -16.796 0.417 1.000 -ATOM 3976 OH2 TIP3 4202 4.020 3.447 -11.354 -0.834 1.520 -ATOM 3977 H1 TIP3 4202 4.595 3.186 -10.658 0.417 1.000 -ATOM 3978 H2 TIP3 4202 3.148 3.534 -10.859 0.417 1.000 -ATOM 3979 OH2 TIP3 4208 16.040 18.645 10.137 -0.834 1.520 -ATOM 3980 H1 TIP3 4208 15.818 18.251 9.289 0.417 1.000 -ATOM 3981 H2 TIP3 4208 16.743 18.081 10.424 0.417 1.000 -ATOM 3982 OH2 TIP3 4209 17.866 16.382 10.694 -0.834 1.520 -ATOM 3983 H1 TIP3 4209 16.987 16.068 10.527 0.417 1.000 -ATOM 3984 H2 TIP3 4209 18.004 16.114 11.611 0.417 1.000 -ATOM 3985 OH2 TIP3 4211 13.191 19.588 12.121 -0.834 1.520 -ATOM 3986 H1 TIP3 4211 13.688 19.512 11.276 0.417 1.000 -ATOM 3987 H2 TIP3 4211 13.852 19.654 12.886 0.417 1.000 -ATOM 3988 OH2 TIP3 4215 5.341 14.047 15.175 -0.834 1.520 -ATOM 3989 H1 TIP3 4215 5.027 14.819 14.579 0.417 1.000 -ATOM 3990 H2 TIP3 4215 5.786 13.510 14.502 0.417 1.000 -ATOM 3991 OH2 TIP3 4219 -4.887 14.268 -12.565 -0.834 1.520 -ATOM 3992 H1 TIP3 4219 -5.450 13.507 -12.345 0.417 1.000 -ATOM 3993 H2 TIP3 4219 -4.811 14.702 -11.685 0.417 1.000 -ATOM 3994 OH2 TIP3 4225 10.575 16.188 -5.603 -0.834 1.520 -ATOM 3995 H1 TIP3 4225 10.493 15.571 -6.392 0.417 1.000 -ATOM 3996 H2 TIP3 4225 10.725 17.080 -6.020 0.417 1.000 -ATOM 3997 OH2 TIP3 4231 9.922 16.955 11.148 -0.834 1.520 -ATOM 3998 H1 TIP3 4231 10.733 16.825 11.611 0.417 1.000 -ATOM 3999 H2 TIP3 4231 9.251 16.787 11.827 0.417 1.000 -ATOM 4000 OH2 TIP3 4235 15.712 12.959 18.833 -0.834 1.520 -ATOM 4001 H1 TIP3 4235 16.574 13.215 19.229 0.417 1.000 -ATOM 4002 H2 TIP3 4235 15.344 13.862 18.696 0.417 1.000 -ATOM 4003 OH2 TIP3 4243 -2.746 8.667 -5.944 -0.834 1.520 -ATOM 4004 H1 TIP3 4243 -2.138 9.322 -5.840 0.417 1.000 -ATOM 4005 H2 TIP3 4243 -2.223 7.880 -6.124 0.417 1.000 -ATOM 4006 OH2 TIP3 4247 7.357 16.032 -11.843 -0.834 1.520 -ATOM 4007 H1 TIP3 4247 7.873 15.271 -11.480 0.417 1.000 -ATOM 4008 H2 TIP3 4247 7.529 15.967 -12.836 0.417 1.000 -ATOM 4009 OH2 TIP3 4251 17.050 15.793 0.041 -0.834 1.520 -ATOM 4010 H1 TIP3 4251 16.956 14.986 -0.518 0.417 1.000 -ATOM 4011 H2 TIP3 4251 16.506 15.560 0.846 0.417 1.000 -ATOM 4012 OH2 TIP3 4266 11.616 19.099 -12.332 -0.834 1.520 -ATOM 4013 H1 TIP3 4266 10.929 18.497 -12.078 0.417 1.000 -ATOM 4014 H2 TIP3 4266 11.948 19.297 -11.435 0.417 1.000 -ATOM 4015 OH2 TIP3 4273 4.693 13.537 2.244 -0.834 1.520 -ATOM 4016 H1 TIP3 4273 4.931 12.583 2.306 0.417 1.000 -ATOM 4017 H2 TIP3 4273 4.874 13.826 3.118 0.417 1.000 -ATOM 4018 OH2 TIP3 4287 -7.859 12.047 -6.794 -0.834 1.520 -ATOM 4019 H1 TIP3 4287 -7.801 12.594 -7.663 0.417 1.000 -ATOM 4020 H2 TIP3 4287 -6.883 12.148 -6.520 0.417 1.000 -ATOM 4021 OH2 TIP3 4304 17.274 16.637 -13.285 -0.834 1.520 -ATOM 4022 H1 TIP3 4304 16.350 17.009 -13.199 0.417 1.000 -ATOM 4023 H2 TIP3 4304 17.787 17.175 -12.693 0.417 1.000 -ATOM 4024 OH2 TIP3 4307 14.858 15.098 -14.371 -0.834 1.520 -ATOM 4025 H1 TIP3 4307 14.077 15.462 -14.005 0.417 1.000 -ATOM 4026 H2 TIP3 4307 15.510 15.851 -14.611 0.417 1.000 -ATOM 4027 OH2 TIP3 4315 2.279 12.287 11.129 -0.834 1.520 -ATOM 4028 H1 TIP3 4315 1.427 12.216 10.699 0.417 1.000 -ATOM 4029 H2 TIP3 4315 2.341 11.392 11.600 0.417 1.000 -ATOM 4030 OH2 TIP3 4356 3.384 -13.603 -0.594 -0.834 1.520 -ATOM 4031 H1 TIP3 4356 3.481 -13.468 0.329 0.417 1.000 -ATOM 4032 H2 TIP3 4356 2.511 -14.056 -0.582 0.417 1.000 -ATOM 4033 OH2 TIP3 4380 -1.964 -18.062 4.191 -0.834 1.520 -ATOM 4034 H1 TIP3 4380 -1.228 -18.081 4.829 0.417 1.000 -ATOM 4035 H2 TIP3 4380 -1.577 -18.545 3.463 0.417 1.000 -ATOM 4036 OH2 TIP3 4394 3.400 -11.608 -9.697 -0.834 1.520 -ATOM 4037 H1 TIP3 4394 4.178 -12.174 -9.711 0.417 1.000 -ATOM 4038 H2 TIP3 4394 3.008 -11.779 -8.782 0.417 1.000 -ATOM 4039 OH2 TIP3 4396 5.172 -12.191 9.494 -0.834 1.520 -ATOM 4040 H1 TIP3 4396 5.176 -11.534 10.186 0.417 1.000 -ATOM 4041 H2 TIP3 4396 5.535 -11.703 8.743 0.417 1.000 -ATOM 4042 OH2 TIP3 4414 10.332 -15.308 -15.952 -0.834 1.520 -ATOM 4043 H1 TIP3 4414 10.893 -16.060 -15.604 0.417 1.000 -ATOM 4044 H2 TIP3 4414 9.553 -15.176 -15.369 0.417 1.000 -ATOM 4045 OH2 TIP3 4417 5.132 -6.845 -1.162 -0.834 1.520 -ATOM 4046 H1 TIP3 4417 4.699 -5.934 -1.280 0.417 1.000 -ATOM 4047 H2 TIP3 4417 4.810 -7.123 -0.285 0.417 1.000 -ATOM 4048 OH2 TIP3 4419 5.503 -17.247 -3.035 -0.834 1.520 -ATOM 4049 H1 TIP3 4419 5.005 -16.406 -3.081 0.417 1.000 -ATOM 4050 H2 TIP3 4419 5.901 -17.218 -3.914 0.417 1.000 -ATOM 4051 OH2 TIP3 4421 9.489 -15.226 11.331 -0.834 1.520 -ATOM 4052 H1 TIP3 4421 10.368 -15.260 11.733 0.417 1.000 -ATOM 4053 H2 TIP3 4421 9.461 -14.616 10.529 0.417 1.000 -ATOM 4054 OH2 TIP3 4422 18.173 -16.870 12.614 -0.834 1.520 -ATOM 4055 H1 TIP3 4422 18.948 -16.846 13.185 0.417 1.000 -ATOM 4056 H2 TIP3 4422 17.924 -15.926 12.518 0.417 1.000 -ATOM 4057 OH2 TIP3 4432 15.217 -1.522 -9.399 -0.834 1.520 -ATOM 4058 H1 TIP3 4432 14.477 -0.993 -9.739 0.417 1.000 -ATOM 4059 H2 TIP3 4432 15.647 -1.909 -10.229 0.417 1.000 -ATOM 4060 OH2 TIP3 4434 16.697 -14.491 -12.843 -0.834 1.520 -ATOM 4061 H1 TIP3 4434 16.091 -15.196 -12.606 0.417 1.000 -ATOM 4062 H2 TIP3 4434 16.834 -14.033 -12.010 0.417 1.000 -ATOM 4063 OH2 TIP3 4437 9.655 -11.328 -18.476 -0.834 1.520 -ATOM 4064 H1 TIP3 4437 9.762 -11.373 -17.509 0.417 1.000 -ATOM 4065 H2 TIP3 4437 9.214 -10.489 -18.681 0.417 1.000 -ATOM 4066 OH2 TIP3 4438 13.579 -1.716 -0.439 -0.834 1.520 -ATOM 4067 H1 TIP3 4438 14.100 -0.964 -0.016 0.417 1.000 -ATOM 4068 H2 TIP3 4438 14.046 -2.482 -0.164 0.417 1.000 -ATOM 4069 OH2 TIP3 4441 16.930 -17.526 10.084 -0.834 1.520 -ATOM 4070 H1 TIP3 4441 17.319 -17.311 10.943 0.417 1.000 -ATOM 4071 H2 TIP3 4441 17.488 -18.263 9.698 0.417 1.000 -ATOM 4072 OH2 TIP3 4442 12.062 -10.642 8.946 -0.834 1.520 -ATOM 4073 H1 TIP3 4442 11.398 -11.189 9.447 0.417 1.000 -ATOM 4074 H2 TIP3 4442 12.665 -10.420 9.672 0.417 1.000 -ATOM 4075 OH2 TIP3 4443 15.166 -7.503 16.394 -0.834 1.520 -ATOM 4076 H1 TIP3 4443 15.812 -6.773 16.164 0.417 1.000 -ATOM 4077 H2 TIP3 4443 14.925 -7.382 17.288 0.417 1.000 -ATOM 4078 OH2 TIP3 4455 7.445 -15.199 -14.707 -0.834 1.520 -ATOM 4079 H1 TIP3 4455 7.476 -16.017 -14.163 0.417 1.000 -ATOM 4080 H2 TIP3 4455 7.046 -14.540 -14.139 0.417 1.000 -ATOM 4081 OH2 TIP3 4456 9.751 -8.283 8.837 -0.834 1.520 -ATOM 4082 H1 TIP3 4456 9.421 -8.425 7.903 0.417 1.000 -ATOM 4083 H2 TIP3 4456 9.205 -7.506 9.067 0.417 1.000 -ATOM 4084 OH2 TIP3 4457 16.162 7.874 -7.021 -0.834 1.520 -ATOM 4085 H1 TIP3 4457 17.054 7.642 -6.800 0.417 1.000 -ATOM 4086 H2 TIP3 4457 15.784 7.060 -7.081 0.417 1.000 -ATOM 4087 OH2 TIP3 4461 11.618 -16.667 0.526 -0.834 1.520 -ATOM 4088 H1 TIP3 4461 11.142 -15.894 0.926 0.417 1.000 -ATOM 4089 H2 TIP3 4461 12.225 -16.274 -0.122 0.417 1.000 -ATOM 4090 OH2 TIP3 4462 14.034 -9.533 10.525 -0.834 1.520 -ATOM 4091 H1 TIP3 4462 14.991 -9.644 10.300 0.417 1.000 -ATOM 4092 H2 TIP3 4462 14.089 -8.835 11.275 0.417 1.000 -ATOM 4093 OH2 TIP3 4465 17.907 -12.350 10.036 -0.834 1.520 -ATOM 4094 H1 TIP3 4465 17.462 -11.513 10.175 0.417 1.000 -ATOM 4095 H2 TIP3 4465 17.511 -12.533 9.159 0.417 1.000 -ATOM 4096 OH2 TIP3 4466 18.553 -13.311 5.369 -0.834 1.520 -ATOM 4097 H1 TIP3 4466 18.108 -13.949 4.739 0.417 1.000 -ATOM 4098 H2 TIP3 4466 19.112 -13.875 5.953 0.417 1.000 -ATOM 4099 OH2 TIP3 4473 17.331 -7.317 -18.582 -0.834 1.520 -ATOM 4100 H1 TIP3 4473 16.873 -7.395 -19.420 0.417 1.000 -ATOM 4101 H2 TIP3 4473 17.504 -6.322 -18.536 0.417 1.000 -ATOM 4102 OH2 TIP3 4474 13.250 12.565 -12.865 -0.834 1.520 -ATOM 4103 H1 TIP3 4474 13.340 11.623 -12.647 0.417 1.000 -ATOM 4104 H2 TIP3 4474 12.707 12.922 -12.188 0.417 1.000 -ATOM 4105 OH2 TIP3 4477 13.966 -14.428 -15.275 -0.834 1.520 -ATOM 4106 H1 TIP3 4477 14.760 -14.591 -15.796 0.417 1.000 -ATOM 4107 H2 TIP3 4477 13.593 -15.326 -14.961 0.417 1.000 -ATOM 4108 OH2 TIP3 4479 12.349 -7.984 -8.548 -0.834 1.520 -ATOM 4109 H1 TIP3 4479 11.652 -8.383 -7.971 0.417 1.000 -ATOM 4110 H2 TIP3 4479 13.149 -8.074 -7.996 0.417 1.000 -ATOM 4111 OH2 TIP3 4480 1.457 -0.096 -1.151 -0.834 1.520 -ATOM 4112 H1 TIP3 4480 0.517 -0.113 -1.045 0.417 1.000 -ATOM 4113 H2 TIP3 4480 1.748 0.201 -0.225 0.417 1.000 -ATOM 4114 OH2 TIP3 4483 18.557 0.554 15.616 -0.834 1.520 -ATOM 4115 H1 TIP3 4483 17.932 0.767 14.896 0.417 1.000 -ATOM 4116 H2 TIP3 4483 18.884 1.395 15.886 0.417 1.000 -ATOM 4117 OH2 TIP3 4484 1.289 4.440 6.231 -0.834 1.520 -ATOM 4118 H1 TIP3 4484 1.646 4.235 5.374 0.417 1.000 -ATOM 4119 H2 TIP3 4484 1.743 5.233 6.585 0.417 1.000 -ATOM 4120 OH2 TIP3 4485 -0.996 -18.120 12.785 -0.834 1.520 -ATOM 4121 H1 TIP3 4485 -1.571 -18.243 11.981 0.417 1.000 -ATOM 4122 H2 TIP3 4485 -0.699 -19.007 13.010 0.417 1.000 -ATOM 4123 OH2 TIP3 4496 13.027 2.407 -7.441 -0.834 1.520 -ATOM 4124 H1 TIP3 4496 12.840 1.573 -7.933 0.417 1.000 -ATOM 4125 H2 TIP3 4496 13.395 2.972 -8.121 0.417 1.000 -ATOM 4126 OH2 TIP3 4497 1.477 18.729 -8.303 -0.834 1.520 -ATOM 4127 H1 TIP3 4497 1.458 18.840 -9.294 0.417 1.000 -ATOM 4128 H2 TIP3 4497 0.942 17.904 -8.111 0.417 1.000 -ATOM 4129 OH2 TIP3 4498 3.539 -4.333 11.422 -0.834 1.520 -ATOM 4130 H1 TIP3 4498 4.312 -3.756 11.445 0.417 1.000 -ATOM 4131 H2 TIP3 4498 2.967 -3.738 10.954 0.417 1.000 -ATOM 4132 OH2 TIP3 4499 17.198 -5.556 -4.175 -0.834 1.520 -ATOM 4133 H1 TIP3 4499 16.753 -5.273 -3.323 0.417 1.000 -ATOM 4134 H2 TIP3 4499 18.019 -5.096 -4.196 0.417 1.000 -ATOM 4135 OH2 TIP3 4500 18.648 -9.065 -16.791 -0.834 1.520 -ATOM 4136 H1 TIP3 4500 18.213 -8.491 -17.462 0.417 1.000 -ATOM 4137 H2 TIP3 4500 18.113 -9.113 -15.992 0.417 1.000 -ATOM 4138 OH2 TIP3 4501 -5.318 -8.666 -5.612 -0.834 1.520 -ATOM 4139 H1 TIP3 4501 -4.764 -8.059 -6.124 0.417 1.000 -ATOM 4140 H2 TIP3 4501 -5.734 -9.317 -6.237 0.417 1.000 -ATOM 4141 OH2 TIP3 4503 4.235 8.471 14.535 -0.834 1.520 -ATOM 4142 H1 TIP3 4503 3.499 8.104 14.977 0.417 1.000 -ATOM 4143 H2 TIP3 4503 4.770 8.494 15.356 0.417 1.000 -ATOM 4144 OH2 TIP3 4509 8.680 -13.583 15.453 -0.834 1.520 -ATOM 4145 H1 TIP3 4509 9.288 -12.944 15.767 0.417 1.000 -ATOM 4146 H2 TIP3 4509 8.525 -14.153 16.240 0.417 1.000 -ATOM 4147 OH2 TIP3 4513 16.686 -4.537 -14.481 -0.834 1.520 -ATOM 4148 H1 TIP3 4513 15.907 -5.023 -14.424 0.417 1.000 -ATOM 4149 H2 TIP3 4513 17.050 -4.941 -15.339 0.417 1.000 -ATOM 4150 OH2 TIP3 4515 -7.586 -10.996 -6.091 -0.834 1.520 -ATOM 4151 H1 TIP3 4515 -8.477 -11.069 -5.816 0.417 1.000 -ATOM 4152 H2 TIP3 4515 -7.625 -10.234 -6.714 0.417 1.000 -ATOM 4153 OH2 TIP3 4516 5.948 5.182 -12.605 -0.834 1.520 -ATOM 4154 H1 TIP3 4516 5.997 4.870 -13.514 0.417 1.000 -ATOM 4155 H2 TIP3 4516 5.233 4.634 -12.130 0.417 1.000 -ATOM 4156 OH2 TIP3 4518 18.146 8.444 -12.676 -0.834 1.520 -ATOM 4157 H1 TIP3 4518 18.822 8.278 -13.319 0.417 1.000 -ATOM 4158 H2 TIP3 4518 18.298 9.354 -12.307 0.417 1.000 -ATOM 4159 OH2 TIP3 4519 -0.929 -2.722 -18.619 -0.834 1.520 -ATOM 4160 H1 TIP3 4519 -1.109 -2.940 -19.564 0.417 1.000 -ATOM 4161 H2 TIP3 4519 -0.115 -3.219 -18.491 0.417 1.000 -ATOM 4162 OH2 TIP3 4520 15.355 2.251 2.902 -0.834 1.520 -ATOM 4163 H1 TIP3 4520 15.943 1.654 3.397 0.417 1.000 -ATOM 4164 H2 TIP3 4520 15.163 3.005 3.523 0.417 1.000 -ATOM 4165 OH2 TIP3 4523 3.954 -7.885 1.190 -0.834 1.520 -ATOM 4166 H1 TIP3 4523 4.341 -8.397 1.865 0.417 1.000 -ATOM 4167 H2 TIP3 4523 3.309 -7.316 1.623 0.417 1.000 -ATOM 4168 OH2 TIP3 4524 10.071 -8.509 -4.229 -0.834 1.520 -ATOM 4169 H1 TIP3 4524 9.660 -7.590 -4.115 0.417 1.000 -ATOM 4170 H2 TIP3 4524 9.290 -9.095 -4.125 0.417 1.000 -ATOM 4171 OH2 TIP3 4536 11.495 0.117 -11.470 -0.834 1.520 -ATOM 4172 H1 TIP3 4536 11.284 0.364 -12.364 0.417 1.000 -ATOM 4173 H2 TIP3 4536 12.453 0.257 -11.456 0.417 1.000 -ATOM 4174 OH2 TIP3 4537 1.619 3.461 -9.753 -0.834 1.520 -ATOM 4175 H1 TIP3 4537 1.079 3.002 -10.402 0.417 1.000 -ATOM 4176 H2 TIP3 4537 1.571 2.891 -8.929 0.417 1.000 -ATOM 4177 OH2 TIP3 4538 10.990 4.095 -13.875 -0.834 1.520 -ATOM 4178 H1 TIP3 4538 10.920 3.621 -12.993 0.417 1.000 -ATOM 4179 H2 TIP3 4538 11.910 3.868 -14.115 0.417 1.000 -ATOM 4180 OH2 TIP3 4539 11.215 2.123 4.084 -0.834 1.520 -ATOM 4181 H1 TIP3 4539 11.124 2.307 3.136 0.417 1.000 -ATOM 4182 H2 TIP3 4539 11.570 2.978 4.423 0.417 1.000 -ATOM 4183 OH2 TIP3 4540 1.263 16.061 -8.655 -0.834 1.520 -ATOM 4184 H1 TIP3 4540 1.685 16.021 -9.517 0.417 1.000 -ATOM 4185 H2 TIP3 4540 0.886 15.164 -8.555 0.417 1.000 -ATOM 4186 OH2 TIP3 4541 13.538 0.062 14.310 -0.834 1.520 -ATOM 4187 H1 TIP3 4541 12.714 -0.002 13.872 0.417 1.000 -ATOM 4188 H2 TIP3 4541 13.228 -0.287 15.171 0.417 1.000 -ATOM 4189 OH2 TIP3 4543 1.784 -1.228 16.445 -0.834 1.520 -ATOM 4190 H1 TIP3 4543 2.589 -1.711 16.608 0.417 1.000 -ATOM 4191 H2 TIP3 4543 1.535 -1.595 15.600 0.417 1.000 -ATOM 4192 OH2 TIP3 4545 16.643 16.724 7.014 -0.834 1.520 -ATOM 4193 H1 TIP3 4545 16.487 16.247 6.224 0.417 1.000 -ATOM 4194 H2 TIP3 4545 17.378 17.320 6.753 0.417 1.000 -ATOM 4195 OH2 TIP3 4549 18.869 -4.390 14.321 -0.834 1.520 -ATOM 4196 H1 TIP3 4549 18.991 -3.706 15.026 0.417 1.000 -ATOM 4197 H2 TIP3 4549 19.652 -4.905 14.400 0.417 1.000 -ATOM 4198 OH2 TIP3 4551 10.956 17.978 16.943 -0.834 1.520 -ATOM 4199 H1 TIP3 4551 11.185 18.227 16.008 0.417 1.000 -ATOM 4200 H2 TIP3 4551 11.668 18.417 17.549 0.417 1.000 -ATOM 4201 OH2 TIP3 4553 2.736 5.421 -14.182 -0.834 1.520 -ATOM 4202 H1 TIP3 4553 2.831 4.722 -14.918 0.417 1.000 -ATOM 4203 H2 TIP3 4553 1.816 5.685 -14.148 0.417 1.000 -ATOM 4204 OH2 TIP3 4558 17.616 2.220 -14.417 -0.834 1.520 -ATOM 4205 H1 TIP3 4558 18.165 1.700 -14.918 0.417 1.000 -ATOM 4206 H2 TIP3 4558 16.925 1.607 -14.167 0.417 1.000 -ATOM 4207 OH2 TIP3 4559 -3.681 4.159 -13.925 -0.834 1.520 -ATOM 4208 H1 TIP3 4559 -3.767 3.434 -13.225 0.417 1.000 -ATOM 4209 H2 TIP3 4559 -3.686 4.950 -13.338 0.417 1.000 -ATOM 4210 OH2 TIP3 4560 15.504 5.791 -0.107 -0.834 1.520 -ATOM 4211 H1 TIP3 4560 15.552 5.869 -1.034 0.417 1.000 -ATOM 4212 H2 TIP3 4560 15.133 6.668 0.099 0.417 1.000 -ATOM 4213 OH2 TIP3 4561 14.553 -1.863 -16.906 -0.834 1.520 -ATOM 4214 H1 TIP3 4561 13.885 -1.466 -17.499 0.417 1.000 -ATOM 4215 H2 TIP3 4561 14.059 -2.536 -16.426 0.417 1.000 -ATOM 4216 OH2 TIP3 4562 16.051 1.549 -4.253 -0.834 1.520 -ATOM 4217 H1 TIP3 4562 16.255 1.107 -5.080 0.417 1.000 -ATOM 4218 H2 TIP3 4562 16.833 1.449 -3.640 0.417 1.000 -ATOM 4219 OH2 TIP3 4563 18.207 12.168 -9.292 -0.834 1.520 -ATOM 4220 H1 TIP3 4563 18.067 12.816 -8.530 0.417 1.000 -ATOM 4221 H2 TIP3 4563 17.394 11.692 -9.219 0.417 1.000 -ATOM 4222 OH2 TIP3 4564 12.982 6.177 2.803 -0.834 1.520 -ATOM 4223 H1 TIP3 4564 12.584 5.651 3.506 0.417 1.000 -ATOM 4224 H2 TIP3 4564 13.497 6.805 3.320 0.417 1.000 -ATOM 4225 OH2 TIP3 4565 5.352 13.771 -3.849 -0.834 1.520 -ATOM 4226 H1 TIP3 4565 4.521 13.423 -3.587 0.417 1.000 -ATOM 4227 H2 TIP3 4565 5.594 14.435 -3.108 0.417 1.000 -ATOM 4228 OH2 TIP3 4566 13.839 9.446 17.961 -0.834 1.520 -ATOM 4229 H1 TIP3 4566 14.468 9.338 18.585 0.417 1.000 -ATOM 4230 H2 TIP3 4566 13.581 10.371 18.122 0.417 1.000 -ATOM 4231 OH2 TIP3 4567 1.876 8.147 15.900 -0.834 1.520 -ATOM 4232 H1 TIP3 4567 1.471 7.689 16.573 0.417 1.000 -ATOM 4233 H2 TIP3 4567 1.293 8.795 15.666 0.417 1.000 -ATOM 4234 OH2 TIP3 4568 11.686 4.015 14.228 -0.834 1.520 -ATOM 4235 H1 TIP3 4568 11.010 4.071 14.907 0.417 1.000 -ATOM 4236 H2 TIP3 4568 11.263 4.222 13.320 0.417 1.000 -ATOM 4237 OH2 TIP3 4569 11.683 -3.200 -6.633 -0.834 1.520 -ATOM 4238 H1 TIP3 4569 12.318 -2.620 -6.224 0.417 1.000 -ATOM 4239 H2 TIP3 4569 11.964 -3.242 -7.618 0.417 1.000 -ATOM 4240 OH2 TIP3 4574 13.189 -1.652 16.419 -0.834 1.520 -ATOM 4241 H1 TIP3 4574 13.381 -2.579 16.463 0.417 1.000 -ATOM 4242 H2 TIP3 4574 12.287 -1.545 16.721 0.417 1.000 -ATOM 4243 OH2 TIP3 4578 3.418 7.184 -11.936 -0.834 1.520 -ATOM 4244 H1 TIP3 4578 3.266 6.649 -11.124 0.417 1.000 -ATOM 4245 H2 TIP3 4578 3.361 6.508 -12.646 0.417 1.000 -ATOM 4246 OH2 TIP3 4581 19.486 -1.804 -8.135 -0.834 1.520 -ATOM 4247 H1 TIP3 4581 19.797 -1.290 -7.392 0.417 1.000 -ATOM 4248 H2 TIP3 4581 19.728 -2.737 -7.866 0.417 1.000 -ATOM 4249 OH2 TIP3 4582 4.820 -1.079 -16.483 -0.834 1.520 -ATOM 4250 H1 TIP3 4582 3.987 -0.765 -16.882 0.417 1.000 -ATOM 4251 H2 TIP3 4582 4.810 -2.035 -16.689 0.417 1.000 -ATOM 4252 OH2 TIP3 4585 18.295 2.659 10.005 -0.834 1.520 -ATOM 4253 H1 TIP3 4585 18.196 2.250 9.129 0.417 1.000 -ATOM 4254 H2 TIP3 4585 17.557 3.254 10.066 0.417 1.000 -ATOM 4255 OH2 TIP3 4587 9.475 6.956 11.202 -0.834 1.520 -ATOM 4256 H1 TIP3 4587 10.155 7.521 11.557 0.417 1.000 -ATOM 4257 H2 TIP3 4587 9.508 7.116 10.262 0.417 1.000 -ATOM 4258 OH2 TIP3 4588 6.245 3.672 4.162 -0.834 1.520 -ATOM 4259 H1 TIP3 4588 6.070 3.578 3.176 0.417 1.000 -ATOM 4260 H2 TIP3 4588 6.837 2.892 4.382 0.417 1.000 -ATOM 4261 OH2 TIP3 4592 8.091 6.947 16.769 -0.834 1.520 -ATOM 4262 H1 TIP3 4592 7.135 6.953 16.658 0.417 1.000 -ATOM 4263 H2 TIP3 4592 8.386 7.119 15.833 0.417 1.000 -ATOM 4264 OH2 TIP3 4599 17.291 7.648 -18.152 -0.834 1.520 -ATOM 4265 H1 TIP3 4599 16.437 8.105 -18.052 0.417 1.000 -ATOM 4266 H2 TIP3 4599 17.939 8.329 -18.400 0.417 1.000 -ATOM 4267 OH2 TIP3 4600 17.000 10.922 -16.293 -0.834 1.520 -ATOM 4268 H1 TIP3 4600 17.827 11.075 -15.857 0.417 1.000 -ATOM 4269 H2 TIP3 4600 16.406 11.698 -16.077 0.417 1.000 -ATOM 4270 OH2 TIP3 4605 11.347 8.907 3.122 -0.834 1.520 -ATOM 4271 H1 TIP3 4605 10.519 8.665 3.592 0.417 1.000 -ATOM 4272 H2 TIP3 4605 11.822 8.024 3.151 0.417 1.000 -ATOM 4273 OH2 TIP3 4606 6.291 -3.276 4.473 -0.834 1.520 -ATOM 4274 H1 TIP3 4606 5.444 -3.323 3.993 0.417 1.000 -ATOM 4275 H2 TIP3 4606 6.927 -3.732 3.904 0.417 1.000 -ATOM 4276 OH2 TIP3 4607 17.015 -2.493 5.985 -0.834 1.520 -ATOM 4277 H1 TIP3 4607 17.239 -2.097 6.919 0.417 1.000 -ATOM 4278 H2 TIP3 4607 17.779 -2.999 5.777 0.417 1.000 -ATOM 4279 OH2 TIP3 4608 2.145 13.333 17.529 -0.834 1.520 -ATOM 4280 H1 TIP3 4608 1.979 12.515 18.001 0.417 1.000 -ATOM 4281 H2 TIP3 4608 2.590 13.056 16.741 0.417 1.000 -ATOM 4282 OH2 TIP3 4623 7.223 19.014 -6.033 -0.834 1.520 -ATOM 4283 H1 TIP3 4623 6.743 19.594 -5.397 0.417 1.000 -ATOM 4284 H2 TIP3 4623 6.850 18.084 -5.856 0.417 1.000 -ATOM 4285 OH2 TIP3 4625 5.350 4.745 -6.566 -0.834 1.520 -ATOM 4286 H1 TIP3 4625 6.202 4.228 -6.613 0.417 1.000 -ATOM 4287 H2 TIP3 4625 5.022 4.517 -5.689 0.417 1.000 -ATOM 4288 OH2 TIP3 4626 -1.263 3.393 6.234 -0.834 1.520 -ATOM 4289 H1 TIP3 4626 -1.210 2.937 7.094 0.417 1.000 -ATOM 4290 H2 TIP3 4626 -0.374 3.733 6.125 0.417 1.000 -ATOM 4291 OH2 TIP3 4627 12.361 16.430 -3.441 -0.834 1.520 -ATOM 4292 H1 TIP3 4627 11.788 16.299 -2.730 0.417 1.000 -ATOM 4293 H2 TIP3 4627 11.832 16.094 -4.210 0.417 1.000 -ATOM 4294 OH2 TIP3 4628 19.527 6.045 -11.367 -0.834 1.520 -ATOM 4295 H1 TIP3 4628 18.938 5.260 -11.381 0.417 1.000 -ATOM 4296 H2 TIP3 4628 19.052 6.684 -11.897 0.417 1.000 -ATOM 4297 OH2 TIP3 4630 7.211 10.812 9.615 -0.834 1.520 -ATOM 4298 H1 TIP3 4630 6.789 10.309 10.269 0.417 1.000 -ATOM 4299 H2 TIP3 4630 7.824 10.221 9.210 0.417 1.000 -ATOM 4300 OH2 TIP3 4633 11.767 19.278 7.157 -0.834 1.520 -ATOM 4301 H1 TIP3 4633 12.726 19.564 7.038 0.417 1.000 -ATOM 4302 H2 TIP3 4633 11.646 19.012 8.054 0.417 1.000 -ATOM 4303 OH2 TIP3 4641 11.046 13.069 -14.847 -0.834 1.520 -ATOM 4304 H1 TIP3 4641 11.689 13.175 -14.057 0.417 1.000 -ATOM 4305 H2 TIP3 4641 10.211 13.605 -14.686 0.417 1.000 -ATOM 4306 OH2 TIP3 4645 8.675 13.919 -10.250 -0.834 1.520 -ATOM 4307 H1 TIP3 4645 8.333 12.994 -10.233 0.417 1.000 -ATOM 4308 H2 TIP3 4645 8.157 14.335 -9.591 0.417 1.000 -ATOM 4309 OH2 TIP3 4649 -1.297 8.268 5.613 -0.834 1.520 -ATOM 4310 H1 TIP3 4649 -1.706 7.411 5.851 0.417 1.000 -ATOM 4311 H2 TIP3 4649 -2.019 8.752 5.155 0.417 1.000 -ATOM 4312 OH2 TIP3 4653 13.095 15.980 10.504 -0.834 1.520 -ATOM 4313 H1 TIP3 4653 13.123 16.489 9.727 0.417 1.000 -ATOM 4314 H2 TIP3 4653 14.024 15.716 10.530 0.417 1.000 -ATOM 4315 OH2 TIP3 4654 -2.599 4.980 19.403 -0.834 1.520 -ATOM 4316 H1 TIP3 4654 -1.645 5.096 19.237 0.417 1.000 -ATOM 4317 H2 TIP3 4654 -3.031 5.296 18.591 0.417 1.000 -ATOM 4318 OH2 TIP3 4663 6.521 7.229 -5.307 -0.834 1.520 -ATOM 4319 H1 TIP3 4663 7.351 7.069 -4.827 0.417 1.000 -ATOM 4320 H2 TIP3 4663 6.583 6.483 -5.937 0.417 1.000 -ATOM 4321 OH2 TIP3 4668 12.950 19.024 -3.862 -0.834 1.520 -ATOM 4322 H1 TIP3 4668 12.841 18.104 -3.603 0.417 1.000 -ATOM 4323 H2 TIP3 4668 12.767 18.892 -4.826 0.417 1.000 -ATOM 4324 OH2 TIP3 4669 -1.709 14.973 11.665 -0.834 1.520 -ATOM 4325 H1 TIP3 4669 -1.975 14.247 11.144 0.417 1.000 -ATOM 4326 H2 TIP3 4669 -1.833 15.703 11.016 0.417 1.000 -ATOM 4327 OH2 TIP3 4672 15.569 14.734 10.645 -0.834 1.520 -ATOM 4328 H1 TIP3 4672 15.624 13.978 11.267 0.417 1.000 -ATOM 4329 H2 TIP3 4672 15.623 14.307 9.760 0.417 1.000 -ATOM 4330 OH2 TIP3 4677 11.446 19.237 14.364 -0.834 1.520 -ATOM 4331 H1 TIP3 4677 12.163 19.684 13.928 0.417 1.000 -ATOM 4332 H2 TIP3 4677 11.222 18.514 13.754 0.417 1.000 -ATOM 4333 OH2 TIP3 4685 12.972 13.480 -7.140 -0.834 1.520 -ATOM 4334 H1 TIP3 4685 12.983 12.627 -6.655 0.417 1.000 -ATOM 4335 H2 TIP3 4685 12.533 13.298 -7.993 0.417 1.000 -ATOM 4336 OH2 TIP3 4687 19.317 17.395 -7.048 -0.834 1.520 -ATOM 4337 H1 TIP3 4687 19.031 17.760 -6.186 0.417 1.000 -ATOM 4338 H2 TIP3 4687 19.054 18.127 -7.599 0.417 1.000 -ATOM 4339 OH2 TIP3 4692 8.687 19.611 17.304 -0.834 1.520 -ATOM 4340 H1 TIP3 4692 9.114 20.000 18.070 0.417 1.000 -ATOM 4341 H2 TIP3 4692 9.403 19.024 16.970 0.417 1.000 -ATOM 4342 OH2 TIP3 4706 5.944 13.478 -6.492 -0.834 1.520 -ATOM 4343 H1 TIP3 4706 5.621 13.604 -5.569 0.417 1.000 -ATOM 4344 H2 TIP3 4706 5.947 12.461 -6.527 0.417 1.000 -ATOM 4345 OH2 TIP3 4712 8.709 15.833 6.169 -0.834 1.520 -ATOM 4346 H1 TIP3 4712 9.149 15.672 6.985 0.417 1.000 -ATOM 4347 H2 TIP3 4712 9.141 16.681 5.896 0.417 1.000 -ATOM 4348 OH2 TIP3 4725 19.120 10.964 -11.883 -0.834 1.520 -ATOM 4349 H1 TIP3 4725 19.066 11.439 -10.987 0.417 1.000 -ATOM 4350 H2 TIP3 4725 18.534 11.537 -12.395 0.417 1.000 -ATOM 4351 OH2 TIP3 4732 19.291 14.450 -13.157 -0.834 1.520 -ATOM 4352 H1 TIP3 4732 18.552 14.975 -13.364 0.417 1.000 -ATOM 4353 H2 TIP3 4732 18.950 13.553 -13.314 0.417 1.000 -ATOM 4354 OH2 TIP3 4751 9.941 18.988 -6.723 -0.834 1.520 -ATOM 4355 H1 TIP3 4751 9.993 19.950 -6.895 0.417 1.000 -ATOM 4356 H2 TIP3 4751 8.990 18.865 -6.474 0.417 1.000 -ATOM 4357 OH2 TIP3 4787 16.528 10.978 -19.727 -0.834 1.520 -ATOM 4358 H1 TIP3 4787 16.655 11.907 -19.596 0.417 1.000 -ATOM 4359 H2 TIP3 4787 17.409 10.582 -19.468 0.417 1.000 -ATOM 4360 OH2 TIP3 4797 6.084 -12.396 4.020 -0.834 1.520 -ATOM 4361 H1 TIP3 4797 6.375 -11.741 3.400 0.417 1.000 -ATOM 4362 H2 TIP3 4797 6.866 -12.894 4.289 0.417 1.000 -ATOM 4363 OH2 TIP3 4798 -0.315 -19.138 2.352 -0.834 1.520 -ATOM 4364 H1 TIP3 4798 0.261 -18.505 2.737 0.417 1.000 -ATOM 4365 H2 TIP3 4798 -0.496 -18.701 1.487 0.417 1.000 -ATOM 4366 OH2 TIP3 4800 16.815 -19.359 15.056 -0.834 1.520 -ATOM 4367 H1 TIP3 4800 16.177 -18.755 14.675 0.417 1.000 -ATOM 4368 H2 TIP3 4800 17.254 -18.790 15.687 0.417 1.000 -ATOM 4369 OH2 TIP3 4801 6.954 -15.501 12.059 -0.834 1.520 -ATOM 4370 H1 TIP3 4801 7.894 -15.679 11.823 0.417 1.000 -ATOM 4371 H2 TIP3 4801 7.130 -14.661 12.490 0.417 1.000 -ATOM 4372 OH2 TIP3 4809 -4.102 -16.707 -18.352 -0.834 1.520 -ATOM 4373 H1 TIP3 4809 -4.163 -16.023 -17.641 0.417 1.000 -ATOM 4374 H2 TIP3 4809 -4.935 -17.204 -18.211 0.417 1.000 -ATOM 4375 OH2 TIP3 4814 10.181 -15.917 3.019 -0.834 1.520 -ATOM 4376 H1 TIP3 4814 11.052 -15.903 3.484 0.417 1.000 -ATOM 4377 H2 TIP3 4814 9.710 -16.569 3.539 0.417 1.000 -ATOM 4378 OH2 TIP3 4815 6.942 -17.643 6.825 -0.834 1.520 -ATOM 4379 H1 TIP3 4815 7.237 -17.643 7.694 0.417 1.000 -ATOM 4380 H2 TIP3 4815 6.453 -16.824 6.797 0.417 1.000 -ATOM 4381 OH2 TIP3 4820 14.901 -16.361 2.978 -0.834 1.520 -ATOM 4382 H1 TIP3 4820 14.587 -17.266 2.878 0.417 1.000 -ATOM 4383 H2 TIP3 4820 14.202 -15.983 3.577 0.417 1.000 -ATOM 4384 OH2 TIP3 4822 14.066 -18.958 4.537 -0.834 1.520 -ATOM 4385 H1 TIP3 4822 13.373 -19.364 4.102 0.417 1.000 -ATOM 4386 H2 TIP3 4822 13.669 -18.607 5.389 0.417 1.000 -ATOM 4387 OH2 TIP3 4834 14.338 -17.051 -19.090 -0.834 1.520 -ATOM 4388 H1 TIP3 4834 15.061 -16.382 -19.191 0.417 1.000 -ATOM 4389 H2 TIP3 4834 13.640 -16.569 -18.852 0.417 1.000 -ATOM 4390 OH2 TIP3 4835 7.681 -17.533 -13.447 -0.834 1.520 -ATOM 4391 H1 TIP3 4835 7.966 -18.270 -14.010 0.417 1.000 -ATOM 4392 H2 TIP3 4835 6.680 -17.588 -13.407 0.417 1.000 -ATOM 4393 OH2 TIP3 4836 19.678 -19.638 -16.511 -0.834 1.520 -ATOM 4394 H1 TIP3 4836 18.916 -19.672 -17.116 0.417 1.000 -ATOM 4395 H2 TIP3 4836 19.424 -19.064 -15.768 0.417 1.000 -ATOM 4396 OH2 TIP3 4837 0.972 -16.585 2.583 -0.834 1.520 -ATOM 4397 H1 TIP3 4837 1.469 -16.490 3.422 0.417 1.000 -ATOM 4398 H2 TIP3 4837 0.577 -15.762 2.554 0.417 1.000 -ATOM 4399 OH2 TIP3 4844 14.826 -18.690 13.180 -0.834 1.520 -ATOM 4400 H1 TIP3 4844 14.018 -18.883 13.661 0.417 1.000 -ATOM 4401 H2 TIP3 4844 14.653 -19.037 12.277 0.417 1.000 -ATOM 4402 OH2 TIP3 4849 -3.492 -10.422 -12.319 -0.834 1.520 -ATOM 4403 H1 TIP3 4849 -3.756 -10.962 -13.066 0.417 1.000 -ATOM 4404 H2 TIP3 4849 -2.696 -9.979 -12.484 0.417 1.000 -ATOM 4405 OH2 TIP3 4853 11.985 -15.201 -18.696 -0.834 1.520 -ATOM 4406 H1 TIP3 4853 11.753 -15.117 -17.729 0.417 1.000 -ATOM 4407 H2 TIP3 4853 11.077 -15.212 -19.151 0.417 1.000 -ATOM 4408 OH2 TIP3 4855 8.176 -5.568 0.160 -0.834 1.520 -ATOM 4409 H1 TIP3 4855 8.459 -6.493 0.186 0.417 1.000 -ATOM 4410 H2 TIP3 4855 8.243 -5.318 1.086 0.417 1.000 -ATOM 4411 OH2 TIP3 4856 2.237 -16.532 5.032 -0.834 1.520 -ATOM 4412 H1 TIP3 4856 3.140 -16.678 4.668 0.417 1.000 -ATOM 4413 H2 TIP3 4856 2.341 -16.158 5.910 0.417 1.000 -ATOM 4414 OH2 TIP3 4857 0.278 -7.466 -2.928 -0.834 1.520 -ATOM 4415 H1 TIP3 4857 -0.085 -8.071 -3.603 0.417 1.000 -ATOM 4416 H2 TIP3 4857 1.131 -7.948 -2.581 0.417 1.000 -ATOM 4417 OH2 TIP3 4863 15.492 -9.378 0.787 -0.834 1.520 -ATOM 4418 H1 TIP3 4863 14.502 -9.437 0.967 0.417 1.000 -ATOM 4419 H2 TIP3 4863 15.510 -9.719 -0.109 0.417 1.000 -ATOM 4420 OH2 TIP3 4866 9.012 -18.103 13.656 -0.834 1.520 -ATOM 4421 H1 TIP3 4866 9.902 -17.834 13.886 0.417 1.000 -ATOM 4422 H2 TIP3 4866 8.536 -17.314 13.821 0.417 1.000 -ATOM 4423 OH2 TIP3 4872 18.466 3.022 -19.640 -0.834 1.520 -ATOM 4424 H1 TIP3 4872 17.949 3.849 -19.687 0.417 1.000 -ATOM 4425 H2 TIP3 4872 17.859 2.522 -19.124 0.417 1.000 -ATOM 4426 OH2 TIP3 4875 12.252 -16.879 -14.270 -0.834 1.520 -ATOM 4427 H1 TIP3 4875 12.352 -17.136 -13.354 0.417 1.000 -ATOM 4428 H2 TIP3 4875 12.769 -17.581 -14.725 0.417 1.000 -ATOM 4429 OH2 TIP3 4880 15.164 -15.557 6.230 -0.834 1.520 -ATOM 4430 H1 TIP3 4880 15.428 -15.364 5.315 0.417 1.000 -ATOM 4431 H2 TIP3 4880 15.256 -16.516 6.288 0.417 1.000 -ATOM 4432 OH2 TIP3 4881 6.908 -3.729 18.788 -0.834 1.520 -ATOM 4433 H1 TIP3 4881 7.561 -4.201 19.350 0.417 1.000 -ATOM 4434 H2 TIP3 4881 7.116 -4.132 17.910 0.417 1.000 -ATOM 4435 OH2 TIP3 4887 3.580 -16.606 16.850 -0.834 1.520 -ATOM 4436 H1 TIP3 4887 4.442 -17.005 16.956 0.417 1.000 -ATOM 4437 H2 TIP3 4887 3.076 -17.309 16.339 0.417 1.000 -ATOM 4438 OH2 TIP3 4893 19.063 -5.903 -9.481 -0.834 1.520 -ATOM 4439 H1 TIP3 4893 18.356 -5.539 -10.039 0.417 1.000 -ATOM 4440 H2 TIP3 4893 18.874 -5.575 -8.584 0.417 1.000 -ATOM 4441 OH2 TIP3 4896 17.541 -12.996 -2.630 -0.834 1.520 -ATOM 4442 H1 TIP3 4896 17.467 -14.000 -2.570 0.417 1.000 -ATOM 4443 H2 TIP3 4896 16.717 -12.816 -3.094 0.417 1.000 -ATOM 4444 OH2 TIP3 4897 14.708 -10.140 -12.260 -0.834 1.520 -ATOM 4445 H1 TIP3 4897 14.416 -11.033 -12.177 0.417 1.000 -ATOM 4446 H2 TIP3 4897 14.188 -9.835 -12.999 0.417 1.000 -ATOM 4447 OH2 TIP3 4899 6.984 -12.999 -3.420 -0.834 1.520 -ATOM 4448 H1 TIP3 4899 7.702 -13.663 -3.242 0.417 1.000 -ATOM 4449 H2 TIP3 4899 6.454 -12.952 -2.596 0.417 1.000 -ATOM 4450 OH2 TIP3 4902 15.760 -1.047 12.086 -0.834 1.520 -ATOM 4451 H1 TIP3 4902 15.409 -1.089 12.967 0.417 1.000 -ATOM 4452 H2 TIP3 4902 16.073 -1.910 11.954 0.417 1.000 -ATOM 4453 OH2 TIP3 4903 10.925 -12.071 13.169 -0.834 1.520 -ATOM 4454 H1 TIP3 4903 11.834 -12.025 12.689 0.417 1.000 -ATOM 4455 H2 TIP3 4903 10.892 -13.033 13.389 0.417 1.000 -ATOM 4456 OH2 TIP3 4908 11.306 -9.511 17.031 -0.834 1.520 -ATOM 4457 H1 TIP3 4908 11.699 -10.124 17.656 0.417 1.000 -ATOM 4458 H2 TIP3 4908 11.748 -9.807 16.200 0.417 1.000 -ATOM 4459 OH2 TIP3 4915 13.152 -18.032 -11.724 -0.834 1.520 -ATOM 4460 H1 TIP3 4915 12.689 -18.766 -12.103 0.417 1.000 -ATOM 4461 H2 TIP3 4915 13.804 -18.441 -11.091 0.417 1.000 -ATOM 4462 OH2 TIP3 4916 7.810 -6.090 -15.817 -0.834 1.520 -ATOM 4463 H1 TIP3 4916 6.862 -6.324 -15.942 0.417 1.000 -ATOM 4464 H2 TIP3 4916 7.831 -5.361 -15.195 0.417 1.000 -ATOM 4465 OH2 TIP3 4920 15.784 8.837 11.326 -0.834 1.520 -ATOM 4466 H1 TIP3 4920 15.880 8.813 12.330 0.417 1.000 -ATOM 4467 H2 TIP3 4920 16.712 8.805 11.078 0.417 1.000 -ATOM 4468 OH2 TIP3 4922 9.133 -3.754 -7.115 -0.834 1.520 -ATOM 4469 H1 TIP3 4922 9.997 -3.574 -6.746 0.417 1.000 -ATOM 4470 H2 TIP3 4922 8.558 -3.181 -6.645 0.417 1.000 -ATOM 4471 OH2 TIP3 4924 14.087 -16.808 -4.614 -0.834 1.520 -ATOM 4472 H1 TIP3 4924 14.562 -16.864 -5.446 0.417 1.000 -ATOM 4473 H2 TIP3 4924 13.131 -16.814 -4.979 0.417 1.000 -ATOM 4474 OH2 TIP3 4927 12.132 -15.237 16.788 -0.834 1.520 -ATOM 4475 H1 TIP3 4927 12.463 -15.616 17.615 0.417 1.000 -ATOM 4476 H2 TIP3 4927 12.851 -14.679 16.463 0.417 1.000 -ATOM 4477 OH2 TIP3 4936 16.482 -2.803 -11.567 -0.834 1.520 -ATOM 4478 H1 TIP3 4936 16.006 -3.568 -11.905 0.417 1.000 -ATOM 4479 H2 TIP3 4936 17.417 -2.974 -11.853 0.417 1.000 -ATOM 4480 OH2 TIP3 4937 5.885 -16.784 -16.798 -0.834 1.520 -ATOM 4481 H1 TIP3 4937 5.219 -17.371 -16.331 0.417 1.000 -ATOM 4482 H2 TIP3 4937 6.368 -16.371 -16.085 0.417 1.000 -ATOM 4483 OH2 TIP3 4941 8.290 -13.344 7.868 -0.834 1.520 -ATOM 4484 H1 TIP3 4941 8.905 -14.035 7.575 0.417 1.000 -ATOM 4485 H2 TIP3 4941 8.550 -12.608 7.268 0.417 1.000 -ATOM 4486 OH2 TIP3 4942 14.994 -12.738 -6.980 -0.834 1.520 -ATOM 4487 H1 TIP3 4942 14.847 -13.672 -6.988 0.417 1.000 -ATOM 4488 H2 TIP3 4942 14.197 -12.325 -6.627 0.417 1.000 -ATOM 4489 OH2 TIP3 4944 6.220 -3.865 7.306 -0.834 1.520 -ATOM 4490 H1 TIP3 4944 5.697 -3.604 6.557 0.417 1.000 -ATOM 4491 H2 TIP3 4944 5.783 -4.714 7.521 0.417 1.000 -ATOM 4492 OH2 TIP3 4945 8.104 -4.320 11.702 -0.834 1.520 -ATOM 4493 H1 TIP3 4945 8.381 -5.232 11.799 0.417 1.000 -ATOM 4494 H2 TIP3 4945 8.618 -3.820 12.297 0.417 1.000 -ATOM 4495 OH2 TIP3 4946 10.007 4.900 6.483 -0.834 1.520 -ATOM 4496 H1 TIP3 4946 9.793 5.366 7.313 0.417 1.000 -ATOM 4497 H2 TIP3 4946 9.198 5.090 5.945 0.417 1.000 -ATOM 4498 OH2 TIP3 4957 17.926 -7.773 -11.989 -0.834 1.520 -ATOM 4499 H1 TIP3 4957 17.544 -7.514 -11.160 0.417 1.000 -ATOM 4500 H2 TIP3 4957 17.780 -8.743 -12.062 0.417 1.000 -ATOM 4501 OH2 TIP3 4961 5.085 -6.609 11.613 -0.834 1.520 -ATOM 4502 H1 TIP3 4961 5.873 -6.654 12.261 0.417 1.000 -ATOM 4503 H2 TIP3 4961 4.631 -5.785 11.857 0.417 1.000 -ATOM 4504 OH2 TIP3 4963 7.297 -8.112 -2.289 -0.834 1.520 -ATOM 4505 H1 TIP3 4963 6.465 -7.570 -1.982 0.417 1.000 -ATOM 4506 H2 TIP3 4963 7.970 -7.402 -2.304 0.417 1.000 -ATOM 4507 OH2 TIP3 4965 19.289 2.791 -3.905 -0.834 1.520 -ATOM 4508 H1 TIP3 4965 19.949 3.383 -4.380 0.417 1.000 -ATOM 4509 H2 TIP3 4965 18.789 2.404 -4.631 0.417 1.000 -ATOM 4510 OH2 TIP3 4966 18.443 -8.035 13.988 -0.834 1.520 -ATOM 4511 H1 TIP3 4966 19.321 -7.601 14.211 0.417 1.000 -ATOM 4512 H2 TIP3 4966 18.386 -8.648 14.744 0.417 1.000 -ATOM 4513 OH2 TIP3 4978 4.570 0.269 1.544 -0.834 1.520 -ATOM 4514 H1 TIP3 4978 3.726 -0.140 1.727 0.417 1.000 -ATOM 4515 H2 TIP3 4978 4.781 -0.075 0.650 0.417 1.000 -ATOM 4516 OH2 TIP3 4979 13.989 10.080 2.223 -0.834 1.520 -ATOM 4517 H1 TIP3 4979 13.320 10.750 2.434 0.417 1.000 -ATOM 4518 H2 TIP3 4979 13.808 9.315 2.785 0.417 1.000 -ATOM 4519 OH2 TIP3 4980 12.368 -11.230 -19.060 -0.834 1.520 -ATOM 4520 H1 TIP3 4980 11.494 -10.971 -18.862 0.417 1.000 -ATOM 4521 H2 TIP3 4980 12.306 -11.907 -19.814 0.417 1.000 -ATOM 4522 OH2 TIP3 4981 0.993 -3.463 -4.159 -0.834 1.520 -ATOM 4523 H1 TIP3 4981 0.642 -4.363 -4.444 0.417 1.000 -ATOM 4524 H2 TIP3 4981 0.414 -2.868 -4.691 0.417 1.000 -ATOM 4525 OH2 TIP3 4982 10.789 4.157 -7.751 -0.834 1.520 -ATOM 4526 H1 TIP3 4982 10.760 4.783 -6.986 0.417 1.000 -ATOM 4527 H2 TIP3 4982 11.472 3.574 -7.415 0.417 1.000 -ATOM 4528 OH2 TIP3 4983 8.327 -3.150 -0.984 -0.834 1.520 -ATOM 4529 H1 TIP3 4983 8.182 -3.978 -0.387 0.417 1.000 -ATOM 4530 H2 TIP3 4983 8.722 -2.509 -0.412 0.417 1.000 -ATOM 4531 OH2 TIP3 4986 9.388 -6.781 12.244 -0.834 1.520 -ATOM 4532 H1 TIP3 4986 10.345 -6.708 12.433 0.417 1.000 -ATOM 4533 H2 TIP3 4986 9.289 -7.679 11.965 0.417 1.000 -ATOM 4534 OH2 TIP3 4988 18.167 -8.265 7.717 -0.834 1.520 -ATOM 4535 H1 TIP3 4988 17.581 -7.550 7.473 0.417 1.000 -ATOM 4536 H2 TIP3 4988 18.911 -8.189 7.124 0.417 1.000 -ATOM 4537 OH2 TIP3 4993 17.076 -7.771 11.236 -0.834 1.520 -ATOM 4538 H1 TIP3 4993 16.652 -7.228 10.558 0.417 1.000 -ATOM 4539 H2 TIP3 4993 17.875 -7.396 11.525 0.417 1.000 -ATOM 4540 OH2 TIP3 4998 18.949 9.719 -19.076 -0.834 1.520 -ATOM 4541 H1 TIP3 4998 19.477 9.290 -19.783 0.417 1.000 -ATOM 4542 H2 TIP3 4998 19.699 10.048 -18.388 0.417 1.000 -ATOM 4543 OH2 TIP3 4999 7.726 5.191 -3.601 -0.834 1.520 -ATOM 4544 H1 TIP3 4999 8.178 5.298 -2.718 0.417 1.000 -ATOM 4545 H2 TIP3 4999 8.294 4.579 -4.079 0.417 1.000 -ATOM 4546 OH2 TIP3 5000 15.828 5.902 -3.040 -0.834 1.520 -ATOM 4547 H1 TIP3 5000 16.702 5.575 -2.835 0.417 1.000 -ATOM 4548 H2 TIP3 5000 15.627 5.529 -3.991 0.417 1.000 -ATOM 4549 OH2 TIP3 5001 12.344 1.914 -4.634 -0.834 1.520 -ATOM 4550 H1 TIP3 5001 13.019 2.333 -4.142 0.417 1.000 -ATOM 4551 H2 TIP3 5001 12.552 2.120 -5.547 0.417 1.000 -ATOM 4552 OH2 TIP3 5003 17.884 2.121 7.196 -0.834 1.520 -ATOM 4553 H1 TIP3 5003 17.038 2.303 7.639 0.417 1.000 -ATOM 4554 H2 TIP3 5003 18.336 3.004 7.246 0.417 1.000 -ATOM 4555 OH2 TIP3 5005 13.251 -0.783 -6.277 -0.834 1.520 -ATOM 4556 H1 TIP3 5005 14.032 -1.407 -6.444 0.417 1.000 -ATOM 4557 H2 TIP3 5005 13.686 0.092 -6.254 0.417 1.000 -ATOM 4558 OH2 TIP3 5006 9.338 -2.691 6.203 -0.834 1.520 -ATOM 4559 H1 TIP3 5006 8.488 -2.614 6.493 0.417 1.000 -ATOM 4560 H2 TIP3 5006 9.794 -1.905 6.527 0.417 1.000 -ATOM 4561 OH2 TIP3 5007 15.647 -8.204 4.938 -0.834 1.520 -ATOM 4562 H1 TIP3 5007 16.395 -7.719 4.692 0.417 1.000 -ATOM 4563 H2 TIP3 5007 15.193 -7.517 5.556 0.417 1.000 -ATOM 4564 OH2 TIP3 5009 8.335 -4.521 2.753 -0.834 1.520 -ATOM 4565 H1 TIP3 5009 8.907 -3.963 3.229 0.417 1.000 -ATOM 4566 H2 TIP3 5009 8.670 -5.410 2.894 0.417 1.000 -ATOM 4567 OH2 TIP3 5021 15.370 11.305 -10.968 -0.834 1.520 -ATOM 4568 H1 TIP3 5021 15.419 10.340 -11.090 0.417 1.000 -ATOM 4569 H2 TIP3 5021 14.481 11.418 -10.629 0.417 1.000 -ATOM 4570 OH2 TIP3 5023 13.717 9.411 -12.949 -0.834 1.520 -ATOM 4571 H1 TIP3 5023 13.171 9.886 -13.598 0.417 1.000 -ATOM 4572 H2 TIP3 5023 14.515 9.289 -13.485 0.417 1.000 -ATOM 4573 OH2 TIP3 5024 13.592 8.925 15.367 -0.834 1.520 -ATOM 4574 H1 TIP3 5024 13.688 9.032 16.297 0.417 1.000 -ATOM 4575 H2 TIP3 5024 13.902 8.036 15.148 0.417 1.000 -ATOM 4576 OH2 TIP3 5025 4.093 8.900 -1.456 -0.834 1.520 -ATOM 4577 H1 TIP3 5025 3.326 9.488 -1.234 0.417 1.000 -ATOM 4578 H2 TIP3 5025 4.228 8.394 -0.588 0.417 1.000 -ATOM 4579 OH2 TIP3 5027 14.817 8.039 8.515 -0.834 1.520 -ATOM 4580 H1 TIP3 5027 14.697 8.968 8.241 0.417 1.000 -ATOM 4581 H2 TIP3 5027 15.366 8.109 9.312 0.417 1.000 -ATOM 4582 OH2 TIP3 5028 9.347 13.879 17.349 -0.834 1.520 -ATOM 4583 H1 TIP3 5028 9.080 14.658 17.778 0.417 1.000 -ATOM 4584 H2 TIP3 5028 9.829 14.268 16.598 0.417 1.000 -ATOM 4585 OH2 TIP3 5029 13.402 12.231 17.747 -0.834 1.520 -ATOM 4586 H1 TIP3 5029 14.266 12.607 18.005 0.417 1.000 -ATOM 4587 H2 TIP3 5029 13.539 12.199 16.755 0.417 1.000 -ATOM 4588 OH2 TIP3 5043 8.719 10.906 -8.479 -0.834 1.520 -ATOM 4589 H1 TIP3 5043 8.395 11.187 -7.588 0.417 1.000 -ATOM 4590 H2 TIP3 5043 9.590 11.280 -8.626 0.417 1.000 -ATOM 4591 OH2 TIP3 5044 11.408 7.285 -10.340 -0.834 1.520 -ATOM 4592 H1 TIP3 5044 11.908 6.745 -9.809 0.417 1.000 -ATOM 4593 H2 TIP3 5044 10.631 7.638 -9.829 0.417 1.000 -ATOM 4594 OH2 TIP3 5046 6.889 8.990 2.856 -0.834 1.520 -ATOM 4595 H1 TIP3 5046 7.254 9.757 3.253 0.417 1.000 -ATOM 4596 H2 TIP3 5046 7.221 8.318 3.456 0.417 1.000 -ATOM 4597 OH2 TIP3 5047 19.227 2.722 17.313 -0.834 1.520 -ATOM 4598 H1 TIP3 5047 18.720 1.889 17.635 0.417 1.000 -ATOM 4599 H2 TIP3 5047 18.931 3.438 17.940 0.417 1.000 -ATOM 4600 OH2 TIP3 5048 16.055 3.112 19.410 -0.834 1.520 -ATOM 4601 H1 TIP3 5048 15.615 3.280 18.585 0.417 1.000 -ATOM 4602 H2 TIP3 5048 17.001 3.373 19.291 0.417 1.000 -ATOM 4603 OH2 TIP3 5049 16.051 11.403 -3.424 -0.834 1.520 -ATOM 4604 H1 TIP3 5049 16.273 11.416 -4.349 0.417 1.000 -ATOM 4605 H2 TIP3 5049 16.095 10.464 -3.224 0.417 1.000 -ATOM 4606 OH2 TIP3 5061 15.796 2.236 -10.640 -0.834 1.520 -ATOM 4607 H1 TIP3 5061 15.374 2.018 -11.509 0.417 1.000 -ATOM 4608 H2 TIP3 5061 16.534 2.827 -10.902 0.417 1.000 -ATOM 4609 OH2 TIP3 5064 14.936 17.763 -11.514 -0.834 1.520 -ATOM 4610 H1 TIP3 5064 14.177 17.204 -11.674 0.417 1.000 -ATOM 4611 H2 TIP3 5064 15.428 17.293 -10.802 0.417 1.000 -ATOM 4612 OH2 TIP3 5068 15.022 15.199 -6.902 -0.834 1.520 -ATOM 4613 H1 TIP3 5068 14.280 14.568 -7.107 0.417 1.000 -ATOM 4614 H2 TIP3 5068 15.067 15.282 -5.904 0.417 1.000 -ATOM 4615 OH2 TIP3 5075 13.456 12.771 14.985 -0.834 1.520 -ATOM 4616 H1 TIP3 5075 14.249 13.023 14.552 0.417 1.000 -ATOM 4617 H2 TIP3 5075 13.004 13.597 15.189 0.417 1.000 -ATOM 4618 OH2 TIP3 5087 18.535 18.225 5.801 -0.834 1.520 -ATOM 4619 H1 TIP3 5087 17.831 18.919 5.666 0.417 1.000 -ATOM 4620 H2 TIP3 5087 18.496 17.617 5.062 0.417 1.000 -ATOM 4621 OH2 TIP3 5088 11.866 5.965 -0.276 -0.834 1.520 -ATOM 4622 H1 TIP3 5088 12.546 5.327 -0.078 0.417 1.000 -ATOM 4623 H2 TIP3 5088 11.891 6.600 0.377 0.417 1.000 -ATOM 4624 OH2 TIP3 5089 11.223 19.196 10.003 -0.834 1.520 -ATOM 4625 H1 TIP3 5089 11.656 19.268 10.849 0.417 1.000 -ATOM 4626 H2 TIP3 5089 10.442 18.601 10.189 0.417 1.000 -ATOM 4627 OH2 TIP3 5093 12.994 15.909 3.631 -0.834 1.520 -ATOM 4628 H1 TIP3 5093 12.468 16.675 3.756 0.417 1.000 -ATOM 4629 H2 TIP3 5093 12.508 15.300 4.092 0.417 1.000 -ATOM 4630 OH2 TIP3 5098 5.320 2.189 15.296 -0.834 1.520 -ATOM 4631 H1 TIP3 5098 4.510 2.651 15.054 0.417 1.000 -ATOM 4632 H2 TIP3 5098 5.992 2.848 15.086 0.417 1.000 -ATOM 4633 OH2 TIP3 5106 14.747 15.091 -4.128 -0.834 1.520 -ATOM 4634 H1 TIP3 5106 13.802 15.371 -3.863 0.417 1.000 -ATOM 4635 H2 TIP3 5106 15.027 14.529 -3.414 0.417 1.000 -ATOM 4636 OH2 TIP3 5115 7.081 10.869 15.522 -0.834 1.520 -ATOM 4637 H1 TIP3 5115 7.940 10.969 15.073 0.417 1.000 -ATOM 4638 H2 TIP3 5115 7.069 10.065 15.990 0.417 1.000 -ATOM 4639 OH2 TIP3 5117 18.195 13.937 2.979 -0.834 1.520 -ATOM 4640 H1 TIP3 5117 19.057 13.521 2.760 0.417 1.000 -ATOM 4641 H2 TIP3 5117 18.395 14.887 3.058 0.417 1.000 -ATOM 4642 OH2 TIP3 5125 18.369 13.410 7.513 -0.834 1.520 -ATOM 4643 H1 TIP3 5125 19.192 13.302 6.954 0.417 1.000 -ATOM 4644 H2 TIP3 5125 18.414 14.305 7.918 0.417 1.000 -ATOM 4645 OH2 TIP3 5127 10.081 17.085 -10.702 -0.834 1.520 -ATOM 4646 H1 TIP3 5127 9.579 17.884 -10.372 0.417 1.000 -ATOM 4647 H2 TIP3 5127 9.330 16.618 -11.111 0.417 1.000 -ATOM 4648 OH2 TIP3 5133 6.160 10.788 -5.788 -0.834 1.520 -ATOM 4649 H1 TIP3 5133 6.076 10.109 -5.087 0.417 1.000 -ATOM 4650 H2 TIP3 5133 6.406 10.281 -6.565 0.417 1.000 -ATOM 4651 OH2 TIP3 5164 14.212 8.724 -5.348 -0.834 1.520 -ATOM 4652 H1 TIP3 5164 15.073 8.458 -5.841 0.417 1.000 -ATOM 4653 H2 TIP3 5164 13.910 9.428 -5.910 0.417 1.000 -ATOM 4654 OH2 TIP3 5165 12.937 19.219 -9.628 -0.834 1.520 -ATOM 4655 H1 TIP3 5165 12.903 18.243 -9.731 0.417 1.000 -ATOM 4656 H2 TIP3 5165 13.607 19.380 -8.943 0.417 1.000 -ATOM 4657 OH2 TIP3 5195 14.264 5.494 7.481 -0.834 1.520 -ATOM 4658 H1 TIP3 5195 14.316 6.376 7.834 0.417 1.000 -ATOM 4659 H2 TIP3 5195 15.178 5.206 7.322 0.417 1.000 -ATOM 4660 OH2 TIP3 5234 7.070 -17.015 -5.288 -0.834 1.520 -ATOM 4661 H1 TIP3 5234 7.044 -16.425 -6.080 0.417 1.000 -ATOM 4662 H2 TIP3 5234 7.302 -17.929 -5.602 0.417 1.000 -ATOM 4663 OH2 TIP3 5251 9.248 -4.914 -17.871 -0.834 1.520 -ATOM 4664 H1 TIP3 5251 8.830 -5.681 -17.466 0.417 1.000 -ATOM 4665 H2 TIP3 5251 9.093 -4.167 -17.226 0.417 1.000 -ATOM 4666 OH2 TIP3 5260 15.782 -18.766 -0.276 -0.834 1.520 -ATOM 4667 H1 TIP3 5260 16.295 -17.968 0.016 0.417 1.000 -ATOM 4668 H2 TIP3 5260 14.976 -18.442 -0.801 0.417 1.000 -ATOM 4669 OH2 TIP3 5267 5.592 -14.400 18.398 -0.834 1.520 -ATOM 4670 H1 TIP3 5267 6.554 -14.388 18.622 0.417 1.000 -ATOM 4671 H2 TIP3 5267 5.611 -14.914 17.533 0.417 1.000 -ATOM 4672 OH2 TIP3 5272 16.512 -12.638 -14.642 -0.834 1.520 -ATOM 4673 H1 TIP3 5272 17.156 -12.776 -15.412 0.417 1.000 -ATOM 4674 H2 TIP3 5272 16.664 -13.354 -14.037 0.417 1.000 -ATOM 4675 OH2 TIP3 5302 16.233 -13.190 7.692 -0.834 1.520 -ATOM 4676 H1 TIP3 5302 16.788 -13.086 6.952 0.417 1.000 -ATOM 4677 H2 TIP3 5302 15.588 -12.445 7.567 0.417 1.000 -ATOM 4678 OH2 TIP3 5317 14.299 -12.929 -18.042 -0.834 1.520 -ATOM 4679 H1 TIP3 5317 13.774 -13.479 -17.515 0.417 1.000 -ATOM 4680 H2 TIP3 5317 13.798 -12.145 -18.284 0.417 1.000 -ATOM 4681 OH2 TIP3 5318 16.674 -10.069 -1.764 -0.834 1.520 -ATOM 4682 H1 TIP3 5318 16.138 -10.904 -1.806 0.417 1.000 -ATOM 4683 H2 TIP3 5318 16.354 -9.497 -2.518 0.417 1.000 -ATOM 4684 OH2 TIP3 5319 13.503 -18.835 -15.677 -0.834 1.520 -ATOM 4685 H1 TIP3 5319 12.690 -19.285 -15.951 0.417 1.000 -ATOM 4686 H2 TIP3 5319 14.005 -19.019 -16.487 0.417 1.000 -ATOM 4687 OH2 TIP3 5320 -2.725 -9.990 -1.303 -0.834 1.520 -ATOM 4688 H1 TIP3 5320 -2.106 -10.416 -1.906 0.417 1.000 -ATOM 4689 H2 TIP3 5320 -3.508 -10.528 -1.340 0.417 1.000 -ATOM 4690 OH2 TIP3 5323 8.656 -13.880 4.596 -0.834 1.520 -ATOM 4691 H1 TIP3 5323 9.522 -13.819 4.203 0.417 1.000 -ATOM 4692 H2 TIP3 5323 8.851 -14.185 5.493 0.417 1.000 -ATOM 4693 OH2 TIP3 5341 0.606 -6.048 -5.389 -0.834 1.520 -ATOM 4694 H1 TIP3 5341 0.573 -6.455 -4.500 0.417 1.000 -ATOM 4695 H2 TIP3 5341 0.964 -6.827 -5.957 0.417 1.000 -ATOM 4696 OH2 TIP3 5343 12.398 -8.093 7.573 -0.834 1.520 -ATOM 4697 H1 TIP3 5343 12.054 -8.864 8.119 0.417 1.000 -ATOM 4698 H2 TIP3 5343 11.596 -7.736 7.295 0.417 1.000 -ATOM 4699 OH2 TIP3 5355 7.127 -19.591 -17.384 -0.834 1.520 -ATOM 4700 H1 TIP3 5355 6.234 -19.768 -17.558 0.417 1.000 -ATOM 4701 H2 TIP3 5355 7.331 -18.758 -17.819 0.417 1.000 -ATOM 4702 OH2 TIP3 5356 1.148 -17.212 -15.264 -0.834 1.520 -ATOM 4703 H1 TIP3 5356 1.231 -16.486 -14.719 0.417 1.000 -ATOM 4704 H2 TIP3 5356 2.035 -17.255 -15.702 0.417 1.000 -ATOM 4705 OH2 TIP3 5358 11.374 0.383 -14.397 -0.834 1.520 -ATOM 4706 H1 TIP3 5358 10.606 0.894 -14.264 0.417 1.000 -ATOM 4707 H2 TIP3 5358 11.018 -0.304 -15.045 0.417 1.000 -ATOM 4708 OH2 TIP3 5360 14.296 -11.536 -9.788 -0.834 1.520 -ATOM 4709 H1 TIP3 5360 13.560 -11.925 -10.346 0.417 1.000 -ATOM 4710 H2 TIP3 5360 14.266 -12.091 -8.966 0.417 1.000 -ATOM 4711 OH2 TIP3 5364 8.474 1.819 -1.619 -0.834 1.520 -ATOM 4712 H1 TIP3 5364 9.448 2.091 -1.431 0.417 1.000 -ATOM 4713 H2 TIP3 5364 8.268 1.210 -0.894 0.417 1.000 -ATOM 4714 OH2 TIP3 5365 14.100 -11.342 7.382 -0.834 1.520 -ATOM 4715 H1 TIP3 5365 14.415 -10.569 6.920 0.417 1.000 -ATOM 4716 H2 TIP3 5365 13.394 -11.030 7.979 0.417 1.000 -ATOM 4717 OH2 TIP3 5366 4.747 -17.126 3.859 -0.834 1.520 -ATOM 4718 H1 TIP3 5366 5.471 -16.512 3.596 0.417 1.000 -ATOM 4719 H2 TIP3 5366 5.220 -17.923 4.213 0.417 1.000 -ATOM 4720 OH2 TIP3 5369 10.185 -15.314 6.606 -0.834 1.520 -ATOM 4721 H1 TIP3 5369 10.174 -16.257 6.251 0.417 1.000 -ATOM 4722 H2 TIP3 5369 11.134 -15.158 6.778 0.417 1.000 -ATOM 4723 OH2 TIP3 5370 14.602 0.297 17.848 -0.834 1.520 -ATOM 4724 H1 TIP3 5370 14.040 -0.269 17.290 0.417 1.000 -ATOM 4725 H2 TIP3 5370 15.219 -0.394 18.269 0.417 1.000 -ATOM 4726 OH2 TIP3 5378 9.237 -7.661 -9.299 -0.834 1.520 -ATOM 4727 H1 TIP3 5378 9.431 -6.727 -9.322 0.417 1.000 -ATOM 4728 H2 TIP3 5378 10.103 -8.052 -9.119 0.417 1.000 -ATOM 4729 OH2 TIP3 5380 19.043 0.480 -15.906 -0.834 1.520 -ATOM 4730 H1 TIP3 5380 19.347 -0.428 -15.843 0.417 1.000 -ATOM 4731 H2 TIP3 5380 19.763 0.868 -16.339 0.417 1.000 -ATOM 4732 OH2 TIP3 5381 12.704 10.573 -18.808 -0.834 1.520 -ATOM 4733 H1 TIP3 5381 12.141 10.374 -19.588 0.417 1.000 -ATOM 4734 H2 TIP3 5381 12.294 11.463 -18.495 0.417 1.000 -ATOM 4735 OH2 TIP3 5384 15.575 -11.671 -4.060 -0.834 1.520 -ATOM 4736 H1 TIP3 5384 14.988 -12.150 -4.630 0.417 1.000 -ATOM 4737 H2 TIP3 5384 15.336 -10.762 -4.136 0.417 1.000 -ATOM 4738 OH2 TIP3 5385 3.796 -5.682 -4.344 -0.834 1.520 -ATOM 4739 H1 TIP3 5385 4.661 -5.967 -4.718 0.417 1.000 -ATOM 4740 H2 TIP3 5385 3.317 -6.505 -4.204 0.417 1.000 -ATOM 4741 OH2 TIP3 5393 4.111 2.000 -13.574 -0.834 1.520 -ATOM 4742 H1 TIP3 5393 3.832 2.636 -12.908 0.417 1.000 -ATOM 4743 H2 TIP3 5393 3.879 2.428 -14.467 0.417 1.000 -ATOM 4744 OH2 TIP3 5396 13.185 -9.125 -14.505 -0.834 1.520 -ATOM 4745 H1 TIP3 5396 12.568 -8.575 -13.964 0.417 1.000 -ATOM 4746 H2 TIP3 5396 12.538 -9.411 -15.227 0.417 1.000 -ATOM 4747 OH2 TIP3 5400 14.596 -2.476 -3.442 -0.834 1.520 -ATOM 4748 H1 TIP3 5400 14.209 -1.722 -3.033 0.417 1.000 -ATOM 4749 H2 TIP3 5400 14.831 -2.220 -4.363 0.417 1.000 -ATOM 4750 OH2 TIP3 5401 11.944 2.445 7.176 -0.834 1.520 -ATOM 4751 H1 TIP3 5401 11.738 3.383 7.005 0.417 1.000 -ATOM 4752 H2 TIP3 5401 11.566 1.938 6.435 0.417 1.000 -ATOM 4753 OH2 TIP3 5403 8.367 -11.027 6.187 -0.834 1.520 -ATOM 4754 H1 TIP3 5403 8.718 -10.131 6.098 0.417 1.000 -ATOM 4755 H2 TIP3 5403 8.562 -11.490 5.346 0.417 1.000 -ATOM 4756 OH2 TIP3 5404 6.690 -6.853 -10.123 -0.834 1.520 -ATOM 4757 H1 TIP3 5404 7.483 -7.422 -9.872 0.417 1.000 -ATOM 4758 H2 TIP3 5404 6.446 -6.469 -9.181 0.417 1.000 -ATOM 4759 OH2 TIP3 5406 11.389 -16.853 14.614 -0.834 1.520 -ATOM 4760 H1 TIP3 5406 11.689 -16.497 15.506 0.417 1.000 -ATOM 4761 H2 TIP3 5406 11.586 -16.125 13.966 0.417 1.000 -ATOM 4762 OH2 TIP3 5408 18.703 5.587 10.663 -0.834 1.520 -ATOM 4763 H1 TIP3 5408 18.843 6.362 11.336 0.417 1.000 -ATOM 4764 H2 TIP3 5408 19.098 4.769 11.070 0.417 1.000 -ATOM 4765 OH2 TIP3 5409 7.434 5.012 12.576 -0.834 1.520 -ATOM 4766 H1 TIP3 5409 7.156 5.482 13.389 0.417 1.000 -ATOM 4767 H2 TIP3 5409 7.941 5.705 12.091 0.417 1.000 -ATOM 4768 OH2 TIP3 5410 14.092 -10.907 17.180 -0.834 1.520 -ATOM 4769 H1 TIP3 5410 13.629 -10.783 16.383 0.417 1.000 -ATOM 4770 H2 TIP3 5410 14.906 -10.408 17.069 0.417 1.000 -ATOM 4771 OH2 TIP3 5418 6.978 4.748 -15.229 -0.834 1.520 -ATOM 4772 H1 TIP3 5418 7.752 5.332 -15.017 0.417 1.000 -ATOM 4773 H2 TIP3 5418 6.598 5.191 -15.955 0.417 1.000 -ATOM 4774 OH2 TIP3 5421 -19.208 8.251 -2.977 -0.834 1.520 -ATOM 4775 H1 TIP3 5421 -19.033 8.811 -3.808 0.417 1.000 -ATOM 4776 H2 TIP3 5421 -19.678 7.534 -3.339 0.417 1.000 -ATOM 4777 OH2 TIP3 5423 16.783 5.581 8.647 -0.834 1.520 -ATOM 4778 H1 TIP3 5423 17.154 6.118 7.942 0.417 1.000 -ATOM 4779 H2 TIP3 5423 17.451 5.726 9.351 0.417 1.000 -ATOM 4780 OH2 TIP3 5424 18.809 2.308 4.480 -0.834 1.520 -ATOM 4781 H1 TIP3 5424 18.892 3.167 4.977 0.417 1.000 -ATOM 4782 H2 TIP3 5424 17.869 2.017 4.797 0.417 1.000 -ATOM 4783 OH2 TIP3 5429 14.462 -4.027 14.451 -0.834 1.520 -ATOM 4784 H1 TIP3 5429 15.148 -3.401 14.534 0.417 1.000 -ATOM 4785 H2 TIP3 5429 14.480 -4.395 15.324 0.417 1.000 -ATOM 4786 OH2 TIP3 5430 11.900 -3.789 -3.199 -0.834 1.520 -ATOM 4787 H1 TIP3 5430 12.581 -3.277 -3.608 0.417 1.000 -ATOM 4788 H2 TIP3 5430 11.064 -3.277 -3.338 0.417 1.000 -ATOM 4789 OH2 TIP3 5440 14.485 -0.646 -12.162 -0.834 1.520 -ATOM 4790 H1 TIP3 5440 15.335 -1.131 -11.960 0.417 1.000 -ATOM 4791 H2 TIP3 5440 14.813 0.175 -12.584 0.417 1.000 -ATOM 4792 OH2 TIP3 5443 12.609 -9.258 -3.624 -0.834 1.520 -ATOM 4793 H1 TIP3 5443 11.791 -8.811 -3.825 0.417 1.000 -ATOM 4794 H2 TIP3 5443 12.237 -9.936 -2.981 0.417 1.000 -ATOM 4795 OH2 TIP3 5444 11.100 3.775 -10.543 -0.834 1.520 -ATOM 4796 H1 TIP3 5444 10.416 4.441 -10.781 0.417 1.000 -ATOM 4797 H2 TIP3 5444 10.933 3.615 -9.633 0.417 1.000 -ATOM 4798 OH2 TIP3 5445 11.210 12.340 -9.099 -0.834 1.520 -ATOM 4799 H1 TIP3 5445 11.732 11.533 -9.460 0.417 1.000 -ATOM 4800 H2 TIP3 5445 11.181 12.865 -9.901 0.417 1.000 -ATOM 4801 OH2 TIP3 5448 14.455 0.732 10.621 -0.834 1.520 -ATOM 4802 H1 TIP3 5448 14.754 0.073 11.257 0.417 1.000 -ATOM 4803 H2 TIP3 5448 14.486 0.203 9.747 0.417 1.000 -ATOM 4804 OH2 TIP3 5454 14.812 -8.573 13.499 -0.834 1.520 -ATOM 4805 H1 TIP3 5454 15.552 -8.093 13.151 0.417 1.000 -ATOM 4806 H2 TIP3 5454 14.951 -8.465 14.455 0.417 1.000 -ATOM 4807 OH2 TIP3 5455 9.007 4.256 16.647 -0.834 1.520 -ATOM 4808 H1 TIP3 5455 8.876 3.559 17.371 0.417 1.000 -ATOM 4809 H2 TIP3 5455 9.861 4.650 16.935 0.417 1.000 -ATOM 4810 OH2 TIP3 5456 12.228 10.627 13.602 -0.834 1.520 -ATOM 4811 H1 TIP3 5456 12.672 11.422 13.899 0.417 1.000 -ATOM 4812 H2 TIP3 5456 12.664 9.883 14.024 0.417 1.000 -ATOM 4813 OH2 TIP3 5462 9.361 6.115 -14.689 -0.834 1.520 -ATOM 4814 H1 TIP3 5462 10.188 5.555 -14.557 0.417 1.000 -ATOM 4815 H2 TIP3 5462 9.590 6.982 -14.224 0.417 1.000 -ATOM 4816 OH2 TIP3 5463 4.980 6.576 -8.654 -0.834 1.520 -ATOM 4817 H1 TIP3 5463 5.270 6.086 -7.833 0.417 1.000 -ATOM 4818 H2 TIP3 5463 4.047 6.280 -8.778 0.417 1.000 -ATOM 4819 OH2 TIP3 5466 12.725 -3.052 -12.017 -0.834 1.520 -ATOM 4820 H1 TIP3 5466 13.444 -3.698 -12.046 0.417 1.000 -ATOM 4821 H2 TIP3 5466 13.212 -2.192 -12.134 0.417 1.000 -ATOM 4822 OH2 TIP3 5468 17.093 11.448 3.705 -0.834 1.520 -ATOM 4823 H1 TIP3 5468 17.843 11.474 4.294 0.417 1.000 -ATOM 4824 H2 TIP3 5468 17.116 12.288 3.230 0.417 1.000 -ATOM 4825 OH2 TIP3 5470 5.477 1.060 12.510 -0.834 1.520 -ATOM 4826 H1 TIP3 5470 5.580 0.888 13.447 0.417 1.000 -ATOM 4827 H2 TIP3 5470 5.397 1.990 12.354 0.417 1.000 -ATOM 4828 OH2 TIP3 5472 1.772 0.721 13.296 -0.834 1.520 -ATOM 4829 H1 TIP3 5472 0.857 0.872 13.293 0.417 1.000 -ATOM 4830 H2 TIP3 5472 2.111 1.563 12.903 0.417 1.000 -ATOM 4831 OH2 TIP3 5473 10.150 -2.350 14.111 -0.834 1.520 -ATOM 4832 H1 TIP3 5473 10.498 -2.236 14.969 0.417 1.000 -ATOM 4833 H2 TIP3 5473 9.443 -1.642 14.097 0.417 1.000 -ATOM 4834 OH2 TIP3 5484 11.221 -0.513 -1.272 -0.834 1.520 -ATOM 4835 H1 TIP3 5484 10.595 -0.924 -0.698 0.417 1.000 -ATOM 4836 H2 TIP3 5484 11.999 -0.958 -1.025 0.417 1.000 -ATOM 4837 OH2 TIP3 5485 10.724 -2.726 3.501 -0.834 1.520 -ATOM 4838 H1 TIP3 5485 10.500 -2.364 4.425 0.417 1.000 -ATOM 4839 H2 TIP3 5485 11.702 -2.759 3.492 0.417 1.000 -ATOM 4840 OH2 TIP3 5489 11.882 10.956 5.165 -0.834 1.520 -ATOM 4841 H1 TIP3 5489 11.001 11.046 5.465 0.417 1.000 -ATOM 4842 H2 TIP3 5489 11.776 10.285 4.455 0.417 1.000 -ATOM 4843 OH2 TIP3 5490 13.347 6.042 12.170 -0.834 1.520 -ATOM 4844 H1 TIP3 5490 13.110 6.803 11.609 0.417 1.000 -ATOM 4845 H2 TIP3 5490 12.595 5.502 12.128 0.417 1.000 -ATOM 4846 OH2 TIP3 5505 12.381 9.281 -3.169 -0.834 1.520 -ATOM 4847 H1 TIP3 5505 11.517 8.803 -3.188 0.417 1.000 -ATOM 4848 H2 TIP3 5505 12.710 9.154 -4.038 0.417 1.000 -ATOM 4849 OH2 TIP3 5508 12.486 16.535 13.272 -0.834 1.520 -ATOM 4850 H1 TIP3 5508 13.209 16.329 13.871 0.417 1.000 -ATOM 4851 H2 TIP3 5508 12.769 16.183 12.425 0.417 1.000 -ATOM 4852 OH2 TIP3 5511 14.369 -7.349 18.864 -0.834 1.520 -ATOM 4853 H1 TIP3 5511 13.425 -7.553 18.748 0.417 1.000 -ATOM 4854 H2 TIP3 5511 14.443 -7.447 19.814 0.417 1.000 -ATOM 4855 OH2 TIP3 5514 14.950 3.794 11.426 -0.834 1.520 -ATOM 4856 H1 TIP3 5514 14.798 4.765 11.501 0.417 1.000 -ATOM 4857 H2 TIP3 5514 14.399 3.413 12.040 0.417 1.000 -ATOM 4858 OH2 TIP3 5522 19.036 14.614 -17.999 -0.834 1.520 -ATOM 4859 H1 TIP3 5522 19.729 13.928 -18.017 0.417 1.000 -ATOM 4860 H2 TIP3 5522 19.474 15.397 -17.626 0.417 1.000 -ATOM 4861 OH2 TIP3 5528 11.135 2.229 -1.321 -0.834 1.520 -ATOM 4862 H1 TIP3 5528 11.278 1.353 -1.767 0.417 1.000 -ATOM 4863 H2 TIP3 5528 12.011 2.599 -1.355 0.417 1.000 -ATOM 4864 OH2 TIP3 5530 12.170 19.039 1.156 -0.834 1.520 -ATOM 4865 H1 TIP3 5530 12.699 18.209 0.987 0.417 1.000 -ATOM 4866 H2 TIP3 5530 12.023 19.346 0.242 0.417 1.000 -ATOM 4867 OH2 TIP3 5531 15.862 14.515 5.077 -0.834 1.520 -ATOM 4868 H1 TIP3 5531 16.165 14.505 4.102 0.417 1.000 -ATOM 4869 H2 TIP3 5531 15.031 14.091 5.011 0.417 1.000 -ATOM 4870 OH2 TIP3 5537 14.539 10.511 9.425 -0.834 1.520 -ATOM 4871 H1 TIP3 5537 14.825 10.055 10.231 0.417 1.000 -ATOM 4872 H2 TIP3 5537 15.302 11.126 9.271 0.417 1.000 -ATOM 4873 OH2 TIP3 5545 9.835 15.598 -13.639 -0.834 1.520 -ATOM 4874 H1 TIP3 5545 9.391 16.405 -13.887 0.417 1.000 -ATOM 4875 H2 TIP3 5545 9.216 14.860 -13.933 0.417 1.000 -ATOM 4876 OH2 TIP3 5549 6.448 12.371 -0.027 -0.834 1.520 -ATOM 4877 H1 TIP3 5549 6.085 11.922 0.689 0.417 1.000 -ATOM 4878 H2 TIP3 5549 5.867 13.101 -0.215 0.417 1.000 -ATOM 4879 OH2 TIP3 5550 17.281 11.681 -5.760 -0.834 1.520 -ATOM 4880 H1 TIP3 5550 17.520 12.442 -6.341 0.417 1.000 -ATOM 4881 H2 TIP3 5550 17.940 11.105 -6.124 0.417 1.000 -ATOM 4882 OH2 TIP3 5551 19.291 12.022 -2.462 -0.834 1.520 -ATOM 4883 H1 TIP3 5551 19.395 11.139 -2.833 0.417 1.000 -ATOM 4884 H2 TIP3 5551 18.973 11.884 -1.530 0.417 1.000 -ATOM 4885 OH2 TIP3 5557 18.997 17.076 18.505 -0.834 1.520 -ATOM 4886 H1 TIP3 5557 19.159 16.530 17.701 0.417 1.000 -ATOM 4887 H2 TIP3 5557 18.170 16.717 18.937 0.417 1.000 -ATOM 4888 OH2 TIP3 5566 11.906 14.524 -11.130 -0.834 1.520 -ATOM 4889 H1 TIP3 5566 11.644 15.279 -10.636 0.417 1.000 -ATOM 4890 H2 TIP3 5566 12.499 14.976 -11.759 0.417 1.000 -ATOM 4891 OH2 TIP3 5570 16.934 17.142 3.738 -0.834 1.520 -ATOM 4892 H1 TIP3 5570 16.356 17.888 3.511 0.417 1.000 -ATOM 4893 H2 TIP3 5570 16.553 16.472 3.181 0.417 1.000 -ATOM 4894 OH2 TIP3 5575 8.382 13.106 5.620 -0.834 1.520 -ATOM 4895 H1 TIP3 5575 8.699 14.015 5.900 0.417 1.000 -ATOM 4896 H2 TIP3 5575 8.951 12.449 6.124 0.417 1.000 -ATOM 4897 OH2 TIP3 5607 10.069 15.308 -17.921 -0.834 1.520 -ATOM 4898 H1 TIP3 5607 10.483 15.292 -18.754 0.417 1.000 -ATOM 4899 H2 TIP3 5607 9.513 14.548 -17.868 0.417 1.000 -ATOM 4900 OH2 TIP3 5610 15.900 16.217 -9.230 -0.834 1.520 -ATOM 4901 H1 TIP3 5610 15.441 15.887 -8.458 0.417 1.000 -ATOM 4902 H2 TIP3 5610 15.819 15.482 -9.883 0.417 1.000 -ATOM 4903 OH2 TIP3 5613 19.591 16.397 2.036 -0.834 1.520 -ATOM 4904 H1 TIP3 5613 18.836 16.506 1.366 0.417 1.000 -ATOM 4905 H2 TIP3 5613 19.452 17.178 2.647 0.417 1.000 -ATOM 4906 OH2 TIP3 5669 18.511 -17.954 -14.452 -0.834 1.520 -ATOM 4907 H1 TIP3 5669 17.889 -17.419 -14.932 0.417 1.000 -ATOM 4908 H2 TIP3 5669 17.999 -18.232 -13.736 0.417 1.000 -ATOM 4909 OH2 TIP3 5676 17.038 -15.763 -9.630 -0.834 1.520 -ATOM 4910 H1 TIP3 5676 17.927 -15.462 -9.940 0.417 1.000 -ATOM 4911 H2 TIP3 5676 16.566 -14.950 -9.774 0.417 1.000 -ATOM 4912 OH2 TIP3 5690 9.290 -15.575 -7.608 -0.834 1.520 -ATOM 4913 H1 TIP3 5690 9.842 -15.903 -8.302 0.417 1.000 -ATOM 4914 H2 TIP3 5690 8.399 -15.987 -7.841 0.417 1.000 -ATOM 4915 OH2 TIP3 5710 15.432 -18.020 -9.677 -0.834 1.520 -ATOM 4916 H1 TIP3 5710 16.211 -17.449 -9.542 0.417 1.000 -ATOM 4917 H2 TIP3 5710 15.507 -18.652 -8.903 0.417 1.000 -ATOM 4918 OH2 TIP3 5713 19.421 -18.813 -3.561 -0.834 1.520 -ATOM 4919 H1 TIP3 5713 19.141 -18.504 -4.484 0.417 1.000 -ATOM 4920 H2 TIP3 5713 19.100 -18.072 -3.044 0.417 1.000 -ATOM 4921 OH2 TIP3 5719 16.797 -17.705 6.462 -0.834 1.520 -ATOM 4922 H1 TIP3 5719 16.744 -18.284 7.169 0.417 1.000 -ATOM 4923 H2 TIP3 5719 17.466 -18.102 5.855 0.417 1.000 -ATOM 4924 OH2 TIP3 5720 17.339 -13.758 0.208 -0.834 1.520 -ATOM 4925 H1 TIP3 5720 17.305 -12.855 -0.236 0.417 1.000 -ATOM 4926 H2 TIP3 5720 17.919 -13.566 0.953 0.417 1.000 -ATOM 4927 OH2 TIP3 5723 6.919 -15.757 3.224 -0.834 1.520 -ATOM 4928 H1 TIP3 5723 7.546 -15.400 3.905 0.417 1.000 -ATOM 4929 H2 TIP3 5723 7.492 -15.921 2.426 0.417 1.000 -ATOM 4930 OH2 TIP3 5736 18.769 -15.698 -4.759 -0.834 1.520 -ATOM 4931 H1 TIP3 5736 19.640 -15.481 -4.423 0.417 1.000 -ATOM 4932 H2 TIP3 5736 18.425 -14.887 -5.162 0.417 1.000 -ATOM 4933 OH2 TIP3 5741 17.717 -6.300 5.977 -0.834 1.520 -ATOM 4934 H1 TIP3 5741 18.622 -6.487 6.236 0.417 1.000 -ATOM 4935 H2 TIP3 5741 17.860 -6.305 5.015 0.417 1.000 -ATOM 4936 OH2 TIP3 5756 18.720 -19.085 -10.042 -0.834 1.520 -ATOM 4937 H1 TIP3 5756 18.543 -18.745 -10.939 0.417 1.000 -ATOM 4938 H2 TIP3 5756 19.041 -18.335 -9.553 0.417 1.000 -ATOM 4939 OH2 TIP3 5760 18.464 -18.489 4.239 -0.834 1.520 -ATOM 4940 H1 TIP3 5760 18.696 -19.225 4.807 0.417 1.000 -ATOM 4941 H2 TIP3 5760 18.900 -18.765 3.388 0.417 1.000 -ATOM 4942 OH2 TIP3 5765 17.890 -18.159 17.115 -0.834 1.520 -ATOM 4943 H1 TIP3 5765 18.680 -18.480 17.633 0.417 1.000 -ATOM 4944 H2 TIP3 5765 17.229 -18.064 17.819 0.417 1.000 -ATOM 4945 OH2 TIP3 5767 16.459 -16.347 0.744 -0.834 1.520 -ATOM 4946 H1 TIP3 5767 15.941 -16.315 1.542 0.417 1.000 -ATOM 4947 H2 TIP3 5767 16.754 -15.402 0.621 0.417 1.000 -ATOM 4948 OH2 TIP3 5768 14.731 -4.777 0.339 -0.834 1.520 -ATOM 4949 H1 TIP3 5768 13.991 -5.085 -0.209 0.417 1.000 -ATOM 4950 H2 TIP3 5768 14.317 -4.875 1.247 0.417 1.000 -ATOM 4951 OH2 TIP3 5777 19.925 -10.137 -19.866 -0.834 1.520 -ATOM 4952 H1 TIP3 5777 19.934 -9.190 -19.832 0.417 1.000 -ATOM 4953 H2 TIP3 5777 18.975 -10.416 -19.797 0.417 1.000 -ATOM 4954 OH2 TIP3 5782 0.842 -6.430 4.883 -0.834 1.520 -ATOM 4955 H1 TIP3 5782 1.084 -5.588 5.335 0.417 1.000 -ATOM 4956 H2 TIP3 5782 -0.020 -6.620 5.201 0.417 1.000 -ATOM 4957 OH2 TIP3 5784 19.719 -4.976 8.581 -0.834 1.520 -ATOM 4958 H1 TIP3 5784 18.720 -5.011 8.595 0.417 1.000 -ATOM 4959 H2 TIP3 5784 19.932 -4.591 7.681 0.417 1.000 -ATOM 4960 OH2 TIP3 5785 8.411 -14.609 18.565 -0.834 1.520 -ATOM 4961 H1 TIP3 5785 8.638 -14.019 19.284 0.417 1.000 -ATOM 4962 H2 TIP3 5785 8.889 -15.369 18.738 0.417 1.000 -ATOM 4963 OH2 TIP3 5791 17.884 -10.183 12.645 -0.834 1.520 -ATOM 4964 H1 TIP3 5791 17.483 -9.764 11.870 0.417 1.000 -ATOM 4965 H2 TIP3 5791 17.993 -9.426 13.219 0.417 1.000 -ATOM 4966 OH2 TIP3 5795 10.434 -9.295 -6.732 -0.834 1.520 -ATOM 4967 H1 TIP3 5795 9.617 -9.807 -6.823 0.417 1.000 -ATOM 4968 H2 TIP3 5795 10.536 -9.247 -5.767 0.417 1.000 -ATOM 4969 OH2 TIP3 5803 11.367 -17.504 8.704 -0.834 1.520 -ATOM 4970 H1 TIP3 5803 11.892 -16.719 9.045 0.417 1.000 -ATOM 4971 H2 TIP3 5803 11.378 -18.082 9.459 0.417 1.000 -ATOM 4972 OH2 TIP3 5804 2.274 -17.000 8.005 -0.834 1.520 -ATOM 4973 H1 TIP3 5804 1.899 -16.933 8.900 0.417 1.000 -ATOM 4974 H2 TIP3 5804 2.349 -17.966 7.824 0.417 1.000 -ATOM 4975 OH2 TIP3 5806 19.017 -14.066 11.834 -0.834 1.520 -ATOM 4976 H1 TIP3 5806 19.443 -14.671 11.239 0.417 1.000 -ATOM 4977 H2 TIP3 5806 18.689 -13.358 11.192 0.417 1.000 -ATOM 4978 OH2 TIP3 5807 17.605 -8.794 16.594 -0.834 1.520 -ATOM 4979 H1 TIP3 5807 18.105 -7.996 16.877 0.417 1.000 -ATOM 4980 H2 TIP3 5807 16.755 -8.377 16.258 0.417 1.000 -ATOM 4981 OH2 TIP3 5818 14.951 9.246 -18.216 -0.834 1.520 -ATOM 4982 H1 TIP3 5818 15.492 9.897 -18.603 0.417 1.000 -ATOM 4983 H2 TIP3 5818 14.064 9.568 -18.502 0.417 1.000 -ATOM 4984 OH2 TIP3 5820 15.560 -9.147 -8.955 -0.834 1.520 -ATOM 4985 H1 TIP3 5820 15.946 -9.694 -8.244 0.417 1.000 -ATOM 4986 H2 TIP3 5820 15.112 -9.865 -9.526 0.417 1.000 -ATOM 4987 OH2 TIP3 5821 12.574 -5.635 -5.510 -0.834 1.520 -ATOM 4988 H1 TIP3 5821 11.973 -6.364 -5.597 0.417 1.000 -ATOM 4989 H2 TIP3 5821 12.103 -4.921 -5.889 0.417 1.000 -ATOM 4990 OH2 TIP3 5824 12.379 -18.522 6.530 -0.834 1.520 -ATOM 4991 H1 TIP3 5824 12.027 -19.442 6.456 0.417 1.000 -ATOM 4992 H2 TIP3 5824 12.226 -18.201 7.410 0.417 1.000 -ATOM 4993 OH2 TIP3 5827 3.893 -3.789 3.211 -0.834 1.520 -ATOM 4994 H1 TIP3 5827 3.768 -4.616 2.714 0.417 1.000 -ATOM 4995 H2 TIP3 5827 3.179 -3.227 3.003 0.417 1.000 -ATOM 4996 OH2 TIP3 5834 14.343 -5.605 11.122 -0.834 1.520 -ATOM 4997 H1 TIP3 5834 14.853 -5.155 11.833 0.417 1.000 -ATOM 4998 H2 TIP3 5834 13.561 -5.864 11.598 0.417 1.000 -ATOM 4999 OH2 TIP3 5839 13.028 -3.293 -15.034 -0.834 1.520 -ATOM 5000 H1 TIP3 5839 12.488 -3.000 -14.308 0.417 1.000 -ATOM 5001 H2 TIP3 5839 13.536 -3.972 -14.551 0.417 1.000 -ATOM 5002 OH2 TIP3 5840 15.922 8.590 -15.299 -0.834 1.520 -ATOM 5003 H1 TIP3 5840 15.174 8.530 -15.914 0.417 1.000 -ATOM 5004 H2 TIP3 5840 16.526 9.316 -15.654 0.417 1.000 -ATOM 5005 OH2 TIP3 5842 11.352 -1.409 8.149 -0.834 1.520 -ATOM 5006 H1 TIP3 5842 11.651 -2.223 8.551 0.417 1.000 -ATOM 5007 H2 TIP3 5842 11.234 -0.748 8.819 0.417 1.000 -ATOM 5008 OH2 TIP3 5844 19.698 -9.445 -0.979 -0.834 1.520 -ATOM 5009 H1 TIP3 5844 19.296 -8.881 -1.619 0.417 1.000 -ATOM 5010 H2 TIP3 5844 18.953 -10.002 -0.668 0.417 1.000 -ATOM 5011 OH2 TIP3 5845 12.946 4.486 5.134 -0.834 1.520 -ATOM 5012 H1 TIP3 5845 12.703 5.071 5.902 0.417 1.000 -ATOM 5013 H2 TIP3 5845 13.913 4.605 5.047 0.417 1.000 -ATOM 5014 OH2 TIP3 5849 6.737 -6.211 13.818 -0.834 1.520 -ATOM 5015 H1 TIP3 5849 7.398 -6.834 13.923 0.417 1.000 -ATOM 5016 H2 TIP3 5849 7.206 -5.396 13.613 0.417 1.000 -ATOM 5017 OH2 TIP3 5860 9.708 -4.715 -9.528 -0.834 1.520 -ATOM 5018 H1 TIP3 5860 9.576 -4.508 -8.567 0.417 1.000 -ATOM 5019 H2 TIP3 5860 10.628 -4.486 -9.646 0.417 1.000 -ATOM 5020 OH2 TIP3 5861 17.443 4.171 -12.189 -0.834 1.520 -ATOM 5021 H1 TIP3 5861 16.966 4.983 -12.404 0.417 1.000 -ATOM 5022 H2 TIP3 5861 17.263 3.629 -12.969 0.417 1.000 -ATOM 5023 OH2 TIP3 5862 17.233 0.009 -12.818 -0.834 1.520 -ATOM 5024 H1 TIP3 5862 17.096 -0.610 -13.493 0.417 1.000 -ATOM 5025 H2 TIP3 5862 17.966 -0.341 -12.270 0.417 1.000 -ATOM 5026 OH2 TIP3 5864 9.752 -0.123 10.311 -0.834 1.520 -ATOM 5027 H1 TIP3 5864 9.124 -0.691 9.764 0.417 1.000 -ATOM 5028 H2 TIP3 5864 9.196 0.589 10.657 0.417 1.000 -ATOM 5029 OH2 TIP3 5865 16.679 -6.995 1.116 -0.834 1.520 -ATOM 5030 H1 TIP3 5865 16.232 -7.906 1.064 0.417 1.000 -ATOM 5031 H2 TIP3 5865 16.396 -6.522 0.329 0.417 1.000 -ATOM 5032 OH2 TIP3 5868 14.660 -1.218 8.378 -0.834 1.520 -ATOM 5033 H1 TIP3 5868 15.453 -1.616 8.804 0.417 1.000 -ATOM 5034 H2 TIP3 5868 13.978 -1.884 8.611 0.417 1.000 -ATOM 5035 OH2 TIP3 5870 1.849 11.211 19.309 -0.834 1.520 -ATOM 5036 H1 TIP3 5870 1.681 10.244 19.348 0.417 1.000 -ATOM 5037 H2 TIP3 5870 2.819 11.222 19.501 0.417 1.000 -ATOM 5038 OH2 TIP3 5881 15.444 -10.107 -15.865 -0.834 1.520 -ATOM 5039 H1 TIP3 5881 14.898 -9.815 -15.167 0.417 1.000 -ATOM 5040 H2 TIP3 5881 15.644 -11.006 -15.452 0.417 1.000 -ATOM 5041 OH2 TIP3 5884 19.276 -15.078 -11.090 -0.834 1.520 -ATOM 5042 H1 TIP3 5884 19.353 -14.334 -11.676 0.417 1.000 -ATOM 5043 H2 TIP3 5884 19.973 -14.892 -10.463 0.417 1.000 -ATOM 5044 OH2 TIP3 5889 16.284 -6.832 -10.052 -0.834 1.520 -ATOM 5045 H1 TIP3 5889 16.271 -7.712 -9.561 0.417 1.000 -ATOM 5046 H2 TIP3 5889 15.534 -6.950 -10.703 0.417 1.000 -ATOM 5047 OH2 TIP3 5890 16.687 -5.256 15.841 -0.834 1.520 -ATOM 5048 H1 TIP3 5890 17.327 -5.249 15.159 0.417 1.000 -ATOM 5049 H2 TIP3 5890 17.077 -4.701 16.527 0.417 1.000 -ATOM 5050 OH2 TIP3 5899 2.157 6.309 -9.345 -0.834 1.520 -ATOM 5051 H1 TIP3 5899 2.113 5.340 -9.410 0.417 1.000 -ATOM 5052 H2 TIP3 5899 1.349 6.567 -9.887 0.417 1.000 -ATOM 5053 OH2 TIP3 5901 1.701 -3.796 -8.187 -0.834 1.520 -ATOM 5054 H1 TIP3 5901 1.240 -4.722 -8.322 0.417 1.000 -ATOM 5055 H2 TIP3 5901 2.146 -4.007 -7.398 0.417 1.000 -ATOM 5056 OH2 TIP3 5906 14.317 8.605 -0.076 -0.834 1.520 -ATOM 5057 H1 TIP3 5906 13.682 9.059 -0.695 0.417 1.000 -ATOM 5058 H2 TIP3 5906 14.052 9.046 0.699 0.417 1.000 -ATOM 5059 OH2 TIP3 5907 19.718 -0.138 6.819 -0.834 1.520 -ATOM 5060 H1 TIP3 5907 19.928 -0.283 7.739 0.417 1.000 -ATOM 5061 H2 TIP3 5907 19.425 0.779 6.771 0.417 1.000 -ATOM 5062 OH2 TIP3 5910 10.644 6.866 8.586 -0.834 1.520 -ATOM 5063 H1 TIP3 5910 11.352 6.691 9.172 0.417 1.000 -ATOM 5064 H2 TIP3 5910 11.140 7.329 7.913 0.417 1.000 -ATOM 5065 OH2 TIP3 5911 14.024 0.067 1.448 -0.834 1.520 -ATOM 5066 H1 TIP3 5911 14.544 0.795 1.708 0.417 1.000 -ATOM 5067 H2 TIP3 5911 13.067 0.502 1.486 0.417 1.000 -ATOM 5068 OH2 TIP3 5913 4.823 3.470 6.798 -0.834 1.520 -ATOM 5069 H1 TIP3 5913 5.330 4.268 6.950 0.417 1.000 -ATOM 5070 H2 TIP3 5913 5.191 3.224 5.984 0.417 1.000 -ATOM 5071 OH2 TIP3 5914 18.471 -9.128 19.883 -0.834 1.520 -ATOM 5072 H1 TIP3 5914 19.390 -9.501 19.765 0.417 1.000 -ATOM 5073 H2 TIP3 5914 18.468 -8.403 19.264 0.417 1.000 -ATOM 5074 OH2 TIP3 5920 14.510 -6.294 -14.442 -0.834 1.520 -ATOM 5075 H1 TIP3 5920 14.762 -6.216 -15.357 0.417 1.000 -ATOM 5076 H2 TIP3 5920 14.115 -7.218 -14.395 0.417 1.000 -ATOM 5077 OH2 TIP3 5924 17.133 6.283 2.439 -0.834 1.520 -ATOM 5078 H1 TIP3 5924 17.505 7.189 2.338 0.417 1.000 -ATOM 5079 H2 TIP3 5924 16.912 6.021 1.476 0.417 1.000 -ATOM 5080 OH2 TIP3 5925 10.116 18.130 -17.236 -0.834 1.520 -ATOM 5081 H1 TIP3 5925 10.739 17.963 -16.532 0.417 1.000 -ATOM 5082 H2 TIP3 5925 9.814 17.221 -17.426 0.417 1.000 -ATOM 5083 OH2 TIP3 5930 10.375 17.848 4.514 -0.834 1.520 -ATOM 5084 H1 TIP3 5930 10.747 18.288 5.337 0.417 1.000 -ATOM 5085 H2 TIP3 5930 10.077 18.539 3.949 0.417 1.000 -ATOM 5086 OH2 TIP3 5932 15.664 4.835 4.312 -0.834 1.520 -ATOM 5087 H1 TIP3 5932 16.209 5.213 3.655 0.417 1.000 -ATOM 5088 H2 TIP3 5932 16.218 5.094 5.054 0.417 1.000 -ATOM 5089 OH2 TIP3 5935 10.481 9.878 11.227 -0.834 1.520 -ATOM 5090 H1 TIP3 5935 11.225 10.170 11.844 0.417 1.000 -ATOM 5091 H2 TIP3 5935 10.797 10.174 10.293 0.417 1.000 -ATOM 5092 OH2 TIP3 5941 15.927 17.566 -3.842 -0.834 1.520 -ATOM 5093 H1 TIP3 5941 15.511 17.790 -2.984 0.417 1.000 -ATOM 5094 H2 TIP3 5941 15.663 16.687 -4.098 0.417 1.000 -ATOM 5095 OH2 TIP3 5945 7.702 9.196 0.160 -0.834 1.520 -ATOM 5096 H1 TIP3 5945 7.123 9.912 -0.191 0.417 1.000 -ATOM 5097 H2 TIP3 5945 7.403 9.079 1.069 0.417 1.000 -ATOM 5098 OH2 TIP3 5972 16.858 7.112 6.018 -0.834 1.520 -ATOM 5099 H1 TIP3 5972 16.455 7.899 6.455 0.417 1.000 -ATOM 5100 H2 TIP3 5972 17.484 7.544 5.453 0.417 1.000 -ATOM 5101 OH2 TIP3 5973 15.696 14.271 14.278 -0.834 1.520 -ATOM 5102 H1 TIP3 5973 15.870 13.468 13.719 0.417 1.000 -ATOM 5103 H2 TIP3 5973 16.506 14.719 14.177 0.417 1.000 -ATOM 5104 OH2 TIP3 5977 18.465 11.788 15.750 -0.834 1.520 -ATOM 5105 H1 TIP3 5977 17.791 11.588 15.036 0.417 1.000 -ATOM 5106 H2 TIP3 5977 18.044 11.301 16.503 0.417 1.000 -ATOM 5107 OH2 TIP3 5989 5.958 17.112 -9.465 -0.834 1.520 -ATOM 5108 H1 TIP3 5989 6.463 17.075 -10.303 0.417 1.000 -ATOM 5109 H2 TIP3 5989 6.119 18.021 -9.170 0.417 1.000 -ATOM 5110 OH2 TIP3 5992 13.061 16.541 0.071 -0.834 1.520 -ATOM 5111 H1 TIP3 5992 12.167 16.450 -0.238 0.417 1.000 -ATOM 5112 H2 TIP3 5992 13.259 15.623 0.332 0.417 1.000 -ATOM 5113 OH2 TIP3 5993 8.263 16.264 9.221 -0.834 1.520 -ATOM 5114 H1 TIP3 5993 9.031 16.326 9.780 0.417 1.000 -ATOM 5115 H2 TIP3 5993 7.811 15.455 9.592 0.417 1.000 -ATOM 5116 OH2 TIP3 5996 16.031 9.340 14.083 -0.834 1.520 -ATOM 5117 H1 TIP3 5996 16.709 8.952 14.620 0.417 1.000 -ATOM 5118 H2 TIP3 5996 15.219 9.300 14.549 0.417 1.000 -ATOM 5119 OH2 TIP3 6012 18.970 19.936 8.056 -0.834 1.520 -ATOM 5120 H1 TIP3 6012 19.867 19.798 8.384 0.417 1.000 -ATOM 5121 H2 TIP3 6012 19.077 19.535 7.154 0.417 1.000 -ATOM 5122 OH2 TIP3 6019 13.835 11.798 -1.720 -0.834 1.520 -ATOM 5123 H1 TIP3 6019 14.717 11.778 -2.057 0.417 1.000 -ATOM 5124 H2 TIP3 6019 13.509 10.949 -2.116 0.417 1.000 -ATOM 5125 OH2 TIP3 6036 18.524 12.307 0.226 -0.834 1.520 -ATOM 5126 H1 TIP3 6036 18.597 12.412 1.225 0.417 1.000 -ATOM 5127 H2 TIP3 6036 17.692 11.827 0.112 0.417 1.000 -ATOM 5128 OH2 TIP3 6039 15.350 14.076 8.081 -0.834 1.520 -ATOM 5129 H1 TIP3 6039 16.016 14.136 7.313 0.417 1.000 -ATOM 5130 H2 TIP3 6039 14.474 14.045 7.573 0.417 1.000 -ATOM 5131 OH2 TIP3 6048 15.236 12.545 -15.055 -0.834 1.520 -ATOM 5132 H1 TIP3 6048 14.374 12.178 -14.836 0.417 1.000 -ATOM 5133 H2 TIP3 6048 15.075 13.510 -14.862 0.417 1.000 -ATOM 5134 OH2 TIP3 6110 18.595 -18.216 -18.755 -0.834 1.520 -ATOM 5135 H1 TIP3 6110 18.500 -18.695 -19.587 0.417 1.000 -ATOM 5136 H2 TIP3 6110 19.426 -17.785 -18.960 0.417 1.000 -ATOM 5137 OH2 TIP3 6117 18.184 18.079 -17.294 -0.834 1.520 -ATOM 5138 H1 TIP3 6117 17.940 17.568 -16.584 0.417 1.000 -ATOM 5139 H2 TIP3 6117 19.167 18.220 -17.182 0.417 1.000 -ATOM 5140 OH2 TIP3 6136 16.949 -10.849 -7.430 -0.834 1.520 -ATOM 5141 H1 TIP3 6136 17.451 -10.866 -8.263 0.417 1.000 -ATOM 5142 H2 TIP3 6136 16.438 -11.693 -7.510 0.417 1.000 -ATOM 5143 OH2 TIP3 6138 12.773 -17.741 -7.363 -0.834 1.520 -ATOM 5144 H1 TIP3 6138 12.793 -18.533 -6.791 0.417 1.000 -ATOM 5145 H2 TIP3 6138 12.105 -17.987 -8.009 0.417 1.000 -ATOM 5146 OH2 TIP3 6139 16.511 -10.540 6.593 -0.834 1.520 -ATOM 5147 H1 TIP3 6139 17.314 -10.950 6.308 0.417 1.000 -ATOM 5148 H2 TIP3 6139 16.610 -9.592 6.226 0.417 1.000 -ATOM 5149 OH2 TIP3 6158 3.865 -18.173 -15.336 -0.834 1.520 -ATOM 5150 H1 TIP3 6158 3.526 -19.028 -15.556 0.417 1.000 -ATOM 5151 H2 TIP3 6158 4.483 -18.343 -14.531 0.417 1.000 -ATOM 5152 OH2 TIP3 6178 17.511 -11.193 2.131 -0.834 1.520 -ATOM 5153 H1 TIP3 6178 18.343 -10.683 1.977 0.417 1.000 -ATOM 5154 H2 TIP3 6178 16.886 -10.520 1.867 0.417 1.000 -ATOM 5155 OH2 TIP3 6183 9.289 -16.850 -10.699 -0.834 1.520 -ATOM 5156 H1 TIP3 6183 9.950 -16.210 -11.070 0.417 1.000 -ATOM 5157 H2 TIP3 6183 8.761 -17.105 -11.479 0.417 1.000 -ATOM 5158 OH2 TIP3 6187 14.008 -18.363 9.864 -0.834 1.520 -ATOM 5159 H1 TIP3 6187 14.988 -18.244 9.729 0.417 1.000 -ATOM 5160 H2 TIP3 6187 13.759 -17.437 9.984 0.417 1.000 -ATOM 5161 OH2 TIP3 6188 16.927 -15.196 15.751 -0.834 1.520 -ATOM 5162 H1 TIP3 6188 17.543 -15.675 16.351 0.417 1.000 -ATOM 5163 H2 TIP3 6188 17.511 -14.574 15.344 0.417 1.000 -ATOM 5164 OH2 TIP3 6195 10.673 -8.999 -16.499 -0.834 1.520 -ATOM 5165 H1 TIP3 6195 10.445 -9.896 -16.093 0.417 1.000 -ATOM 5166 H2 TIP3 6195 9.950 -8.827 -17.173 0.417 1.000 -ATOM 5167 OH2 TIP3 6198 19.062 -5.607 -13.274 -0.834 1.520 -ATOM 5168 H1 TIP3 6198 18.900 -6.403 -12.820 0.417 1.000 -ATOM 5169 H2 TIP3 6198 18.341 -5.478 -13.914 0.417 1.000 -ATOM 5170 OH2 TIP3 6200 18.172 -18.411 -6.104 -0.834 1.520 -ATOM 5171 H1 TIP3 6200 17.963 -17.499 -6.047 0.417 1.000 -ATOM 5172 H2 TIP3 6200 17.215 -18.764 -6.166 0.417 1.000 -ATOM 5173 OH2 TIP3 6219 18.442 -14.018 -16.211 -0.834 1.520 -ATOM 5174 H1 TIP3 6219 18.510 -14.805 -15.620 0.417 1.000 -ATOM 5175 H2 TIP3 6219 19.384 -14.022 -16.506 0.417 1.000 -ATOM 5176 OH2 TIP3 6220 12.627 0.031 -9.044 -0.834 1.520 -ATOM 5177 H1 TIP3 6220 11.932 -0.017 -9.766 0.417 1.000 -ATOM 5178 H2 TIP3 6220 12.516 -0.821 -8.536 0.417 1.000 -ATOM 5179 OH2 TIP3 6240 -16.920 -19.051 -8.212 -0.834 1.520 -ATOM 5180 H1 TIP3 6240 -16.310 -19.500 -8.828 0.417 1.000 -ATOM 5181 H2 TIP3 6240 -17.804 -19.444 -8.359 0.417 1.000 -ATOM 5182 OH2 TIP3 6241 9.462 1.024 -9.649 -0.834 1.520 -ATOM 5183 H1 TIP3 6241 9.814 0.763 -10.497 0.417 1.000 -ATOM 5184 H2 TIP3 6241 9.452 0.119 -9.241 0.417 1.000 -ATOM 5185 OH2 TIP3 6243 19.114 -10.058 -3.913 -0.834 1.520 -ATOM 5186 H1 TIP3 6243 18.333 -10.698 -3.714 0.417 1.000 -ATOM 5187 H2 TIP3 6243 19.792 -10.607 -3.510 0.417 1.000 -ATOM 5188 OH2 TIP3 6247 9.430 0.387 14.216 -0.834 1.520 -ATOM 5189 H1 TIP3 6247 9.429 1.378 14.066 0.417 1.000 -ATOM 5190 H2 TIP3 6247 10.158 0.138 13.637 0.417 1.000 -ATOM 5191 OH2 TIP3 6256 18.710 -11.190 -9.476 -0.834 1.520 -ATOM 5192 H1 TIP3 6256 19.285 -10.435 -9.422 0.417 1.000 -ATOM 5193 H2 TIP3 6256 19.249 -11.946 -9.577 0.417 1.000 -ATOM 5194 OH2 TIP3 6258 14.517 -5.304 -8.677 -0.834 1.520 -ATOM 5195 H1 TIP3 6258 14.017 -5.790 -8.053 0.417 1.000 -ATOM 5196 H2 TIP3 6258 15.388 -5.778 -8.775 0.417 1.000 -ATOM 5197 OH2 TIP3 6266 13.874 -10.066 3.981 -0.834 1.520 -ATOM 5198 H1 TIP3 6266 14.466 -9.322 4.250 0.417 1.000 -ATOM 5199 H2 TIP3 6266 14.188 -10.361 3.145 0.417 1.000 -ATOM 5200 OH2 TIP3 6300 8.603 5.046 -11.244 -0.834 1.520 -ATOM 5201 H1 TIP3 6300 8.196 5.721 -10.650 0.417 1.000 -ATOM 5202 H2 TIP3 6300 8.345 5.319 -12.104 0.417 1.000 -ATOM 5203 OH2 TIP3 6302 15.255 17.615 -1.139 -0.834 1.520 -ATOM 5204 H1 TIP3 6302 15.637 16.714 -1.125 0.417 1.000 -ATOM 5205 H2 TIP3 6302 14.346 17.487 -0.863 0.417 1.000 -ATOM 5206 OH2 TIP3 6309 15.779 3.135 0.132 -0.834 1.520 -ATOM 5207 H1 TIP3 6309 15.918 2.804 1.032 0.417 1.000 -ATOM 5208 H2 TIP3 6309 15.559 4.079 0.225 0.417 1.000 -ATOM 5209 OH2 TIP3 6311 14.117 -15.216 -7.681 -0.834 1.520 -ATOM 5210 H1 TIP3 6311 13.538 -15.951 -7.665 0.417 1.000 -ATOM 5211 H2 TIP3 6311 14.966 -15.599 -7.412 0.417 1.000 -ATOM 5212 OH2 TIP3 6315 13.898 -12.433 19.265 -0.834 1.520 -ATOM 5213 H1 TIP3 6315 14.046 -11.787 18.505 0.417 1.000 -ATOM 5214 H2 TIP3 6315 14.292 -13.272 18.976 0.417 1.000 -ATOM 5215 OH2 TIP3 6321 16.759 -6.142 -6.759 -0.834 1.520 -ATOM 5216 H1 TIP3 6321 17.254 -6.922 -6.700 0.417 1.000 -ATOM 5217 H2 TIP3 6321 16.903 -5.723 -5.906 0.417 1.000 -ATOM 5218 OH2 TIP3 6325 7.495 0.734 -3.941 -0.834 1.520 -ATOM 5219 H1 TIP3 6325 7.618 1.083 -3.066 0.417 1.000 -ATOM 5220 H2 TIP3 6325 6.546 0.766 -4.029 0.417 1.000 -ATOM 5221 OH2 TIP3 6333 14.076 -4.182 17.215 -0.834 1.520 -ATOM 5222 H1 TIP3 6333 13.717 -4.418 18.037 0.417 1.000 -ATOM 5223 H2 TIP3 6333 15.017 -4.453 17.223 0.417 1.000 -ATOM 5224 OH2 TIP3 6347 17.415 8.449 -0.750 -0.834 1.520 -ATOM 5225 H1 TIP3 6347 16.432 8.260 -0.793 0.417 1.000 -ATOM 5226 H2 TIP3 6347 17.695 8.409 -1.641 0.417 1.000 -ATOM 5227 OH2 TIP3 6348 18.265 5.005 -7.753 -0.834 1.520 -ATOM 5228 H1 TIP3 6348 18.893 4.231 -8.109 0.417 1.000 -ATOM 5229 H2 TIP3 6348 17.406 4.598 -7.627 0.417 1.000 -ATOM 5230 OH2 TIP3 6352 12.470 12.566 2.559 -0.834 1.520 -ATOM 5231 H1 TIP3 6352 11.584 12.597 2.092 0.417 1.000 -ATOM 5232 H2 TIP3 6352 12.196 12.410 3.533 0.417 1.000 -ATOM 5233 OH2 TIP3 6356 13.430 5.959 15.069 -0.834 1.520 -ATOM 5234 H1 TIP3 6356 12.749 5.332 15.250 0.417 1.000 -ATOM 5235 H2 TIP3 6356 13.448 6.034 14.131 0.417 1.000 -ATOM 5236 OH2 TIP3 6370 16.417 0.527 4.809 -0.834 1.520 -ATOM 5237 H1 TIP3 6370 15.913 1.083 5.518 0.417 1.000 -ATOM 5238 H2 TIP3 6370 16.555 -0.381 5.118 0.417 1.000 -ATOM 5239 OH2 TIP3 6373 17.481 9.369 8.871 -0.834 1.520 -ATOM 5240 H1 TIP3 6373 16.902 9.820 8.223 0.417 1.000 -ATOM 5241 H2 TIP3 6373 18.334 9.258 8.353 0.417 1.000 -ATOM 5242 OH2 TIP3 6381 15.388 -5.799 -17.128 -0.834 1.520 -ATOM 5243 H1 TIP3 6381 15.822 -6.583 -17.414 0.417 1.000 -ATOM 5244 H2 TIP3 6381 15.683 -5.117 -17.748 0.417 1.000 -ATOM 5245 OH2 TIP3 6404 14.627 19.477 -18.971 -0.834 1.520 -ATOM 5246 H1 TIP3 6404 15.534 19.676 -18.599 0.417 1.000 -ATOM 5247 H2 TIP3 6404 14.656 19.858 -19.872 0.417 1.000 -ATOM 5248 OH2 TIP3 6417 16.099 11.041 0.540 -0.834 1.520 -ATOM 5249 H1 TIP3 6417 15.853 10.544 -0.247 0.417 1.000 -ATOM 5250 H2 TIP3 6417 15.397 10.963 1.168 0.417 1.000 -ATOM 5251 OH2 TIP3 6425 14.706 13.591 -18.454 -0.834 1.520 -ATOM 5252 H1 TIP3 6425 15.001 13.098 -17.650 0.417 1.000 -ATOM 5253 H2 TIP3 6425 13.797 13.825 -18.206 0.417 1.000 -ATOM 5254 OH2 TIP3 6429 16.114 8.527 -10.530 -0.834 1.520 -ATOM 5255 H1 TIP3 6429 16.585 8.342 -9.719 0.417 1.000 -ATOM 5256 H2 TIP3 6429 16.687 8.403 -11.276 0.417 1.000 -ATOM 5257 OH2 TIP3 6439 16.110 12.348 12.423 -0.834 1.520 -ATOM 5258 H1 TIP3 6439 16.368 12.137 11.512 0.417 1.000 -ATOM 5259 H2 TIP3 6439 16.051 11.474 12.873 0.417 1.000 -ATOM 5260 OH2 TIP3 6456 13.464 13.208 5.608 -0.834 1.520 -ATOM 5261 H1 TIP3 6456 13.071 12.324 5.540 0.417 1.000 -ATOM 5262 H2 TIP3 6456 12.710 13.743 5.337 0.417 1.000 -ATOM 5263 OH2 TIP3 6483 18.297 15.389 13.797 -0.834 1.520 -ATOM 5264 H1 TIP3 6483 18.161 16.339 14.125 0.417 1.000 -ATOM 5265 H2 TIP3 6483 18.816 14.979 14.506 0.417 1.000 -ATOM 5266 OH2 TIP3 6492 10.201 8.396 -5.748 -0.834 1.520 -ATOM 5267 H1 TIP3 6492 9.790 9.206 -5.358 0.417 1.000 -ATOM 5268 H2 TIP3 6492 10.307 7.838 -4.965 0.417 1.000 -ATOM 5269 OH2 TIP3 6499 15.826 8.528 -3.171 -0.834 1.520 -ATOM 5270 H1 TIP3 6499 15.546 7.563 -2.983 0.417 1.000 -ATOM 5271 H2 TIP3 6499 15.225 8.805 -3.879 0.417 1.000 -ATOM 5272 OH2 TIP3 6502 18.029 18.224 13.986 -0.834 1.520 -ATOM 5273 H1 TIP3 6502 18.944 18.397 14.377 0.417 1.000 -ATOM 5274 H2 TIP3 6502 17.985 18.806 13.217 0.417 1.000 -ATOM 5275 OH2 TIP3 6551 17.038 -12.756 -18.318 -0.834 1.520 -ATOM 5276 H1 TIP3 6551 16.056 -12.714 -18.143 0.417 1.000 -ATOM 5277 H2 TIP3 6551 17.432 -13.299 -17.555 0.417 1.000 -ATOM 5278 OH2 TIP3 6575 16.685 -13.065 -10.242 -0.834 1.520 -ATOM 5279 H1 TIP3 6575 15.864 -12.581 -10.217 0.417 1.000 -ATOM 5280 H2 TIP3 6575 17.340 -12.414 -10.034 0.417 1.000 -ATOM 5281 OH2 TIP3 6577 18.667 -17.062 -1.377 -0.834 1.520 -ATOM 5282 H1 TIP3 6577 18.061 -16.505 -1.934 0.417 1.000 -ATOM 5283 H2 TIP3 6577 18.224 -17.043 -0.502 0.417 1.000 -ATOM 5284 OH2 TIP3 6578 18.057 -15.886 3.645 -0.834 1.520 -ATOM 5285 H1 TIP3 6578 17.174 -16.221 3.329 0.417 1.000 -ATOM 5286 H2 TIP3 6578 18.489 -16.639 4.159 0.417 1.000 -ATOM 5287 OH2 TIP3 6583 5.431 -14.780 6.033 -0.834 1.520 -ATOM 5288 H1 TIP3 6583 5.686 -13.883 5.671 0.417 1.000 -ATOM 5289 H2 TIP3 6583 5.655 -14.734 6.978 0.417 1.000 -ATOM 5290 OH2 TIP3 6597 16.934 -18.378 -12.167 -0.834 1.520 -ATOM 5291 H1 TIP3 6597 16.171 -18.749 -12.620 0.417 1.000 -ATOM 5292 H2 TIP3 6597 16.578 -17.959 -11.361 0.417 1.000 -ATOM 5293 OH2 TIP3 6603 18.685 -7.603 -2.963 -0.834 1.520 -ATOM 5294 H1 TIP3 6603 17.893 -7.205 -3.360 0.417 1.000 -ATOM 5295 H2 TIP3 6603 18.756 -8.437 -3.418 0.417 1.000 -ATOM 5296 OH2 TIP3 6623 15.905 3.289 -7.823 -0.834 1.520 -ATOM 5297 H1 TIP3 6623 15.405 2.453 -7.737 0.417 1.000 -ATOM 5298 H2 TIP3 6623 16.099 3.300 -8.793 0.417 1.000 -ATOM 5299 OH2 TIP3 6661 16.588 -1.763 -14.695 -0.834 1.520 -ATOM 5300 H1 TIP3 6661 16.139 -1.597 -15.521 0.417 1.000 -ATOM 5301 H2 TIP3 6661 16.515 -2.721 -14.635 0.417 1.000 -ATOM 5302 OH2 TIP3 6663 16.714 -10.070 9.636 -0.834 1.520 -ATOM 5303 H1 TIP3 6663 17.136 -9.185 9.734 0.417 1.000 -ATOM 5304 H2 TIP3 6663 16.615 -10.113 8.679 0.417 1.000 -ATOM 5305 OH2 TIP3 6667 14.995 -17.815 18.545 -0.834 1.520 -ATOM 5306 H1 TIP3 6667 14.341 -18.180 17.916 0.417 1.000 -ATOM 5307 H2 TIP3 6667 14.462 -17.068 18.949 0.417 1.000 -ATOM 5308 OH2 TIP3 6669 15.835 -13.074 12.145 -0.834 1.520 -ATOM 5309 H1 TIP3 6669 16.636 -13.402 11.616 0.417 1.000 -ATOM 5310 H2 TIP3 6669 15.634 -13.890 12.729 0.417 1.000 -ATOM 5311 OH2 TIP3 6671 -19.103 -7.187 0.940 -0.834 1.520 -ATOM 5312 H1 TIP3 6671 -18.766 -6.423 1.489 0.417 1.000 -ATOM 5313 H2 TIP3 6671 -18.890 -7.917 1.442 0.417 1.000 -ATOM 5314 OH2 TIP3 6672 12.685 -10.560 14.879 -0.834 1.520 -ATOM 5315 H1 TIP3 6672 12.047 -11.075 14.323 0.417 1.000 -ATOM 5316 H2 TIP3 6672 13.281 -10.201 14.205 0.417 1.000 -ATOM 5317 OH2 TIP3 6681 16.626 0.950 -18.215 -0.834 1.520 -ATOM 5318 H1 TIP3 6681 16.391 1.874 -17.889 0.417 1.000 -ATOM 5319 H2 TIP3 6681 17.091 0.565 -17.414 0.417 1.000 -ATOM 5320 OH2 TIP3 6684 7.912 -9.027 -15.928 -0.834 1.520 -ATOM 5321 H1 TIP3 6684 6.970 -9.310 -15.925 0.417 1.000 -ATOM 5322 H2 TIP3 6684 7.786 -8.059 -16.065 0.417 1.000 -ATOM 5323 OH2 TIP3 6703 14.408 -7.477 -6.332 -0.834 1.520 -ATOM 5324 H1 TIP3 6703 14.017 -6.689 -6.042 0.417 1.000 -ATOM 5325 H2 TIP3 6703 15.307 -7.120 -6.630 0.417 1.000 -ATOM 5326 OH2 TIP3 6705 14.244 2.149 -2.137 -0.834 1.520 -ATOM 5327 H1 TIP3 6705 14.911 2.111 -1.431 0.417 1.000 -ATOM 5328 H2 TIP3 6705 14.746 1.912 -2.910 0.417 1.000 -ATOM 5329 OH2 TIP3 6707 15.347 -15.272 9.772 -0.834 1.520 -ATOM 5330 H1 TIP3 6707 15.928 -14.672 9.283 0.417 1.000 -ATOM 5331 H2 TIP3 6707 15.826 -16.067 9.694 0.417 1.000 -ATOM 5332 OH2 TIP3 6726 -1.987 -3.388 -1.134 -0.834 1.520 -ATOM 5333 H1 TIP3 6726 -1.039 -3.237 -1.130 0.417 1.000 -ATOM 5334 H2 TIP3 6726 -2.097 -4.148 -1.767 0.417 1.000 -ATOM 5335 OH2 TIP3 6751 -16.952 6.445 12.140 -0.834 1.520 -ATOM 5336 H1 TIP3 6751 -16.619 5.736 12.733 0.417 1.000 -ATOM 5337 H2 TIP3 6751 -17.900 6.429 12.339 0.417 1.000 -ATOM 5338 OH2 TIP3 6774 8.445 8.554 8.432 -0.834 1.520 -ATOM 5339 H1 TIP3 6774 9.322 8.154 8.374 0.417 1.000 -ATOM 5340 H2 TIP3 6774 7.812 7.759 8.510 0.417 1.000 -ATOM 5341 OH2 TIP3 6813 13.325 -5.870 -2.151 -0.834 1.520 -ATOM 5342 H1 TIP3 6813 13.550 -6.514 -2.851 0.417 1.000 -ATOM 5343 H2 TIP3 6813 12.837 -5.114 -2.565 0.417 1.000 -ATOM 5344 OH2 TIP3 6834 5.747 14.167 -16.239 -0.834 1.520 -ATOM 5345 H1 TIP3 6834 5.515 14.736 -17.052 0.417 1.000 -ATOM 5346 H2 TIP3 6834 4.967 14.186 -15.602 0.417 1.000 -ATOM 5347 OH2 TIP3 6837 18.348 0.190 12.227 -0.834 1.520 -ATOM 5348 H1 TIP3 6837 18.102 1.040 11.791 0.417 1.000 -ATOM 5349 H2 TIP3 6837 17.479 -0.146 12.542 0.417 1.000 -ATOM 5350 OH2 TIP3 6854 16.928 14.027 -1.992 -0.834 1.520 -ATOM 5351 H1 TIP3 6854 16.850 13.124 -2.383 0.417 1.000 -ATOM 5352 H2 TIP3 6854 17.662 14.429 -2.485 0.417 1.000 -ATOM 5353 OH2 TIP3 6855 13.240 3.893 0.917 -0.834 1.520 -ATOM 5354 H1 TIP3 6855 13.505 4.515 1.564 0.417 1.000 -ATOM 5355 H2 TIP3 6855 13.133 3.113 1.432 0.417 1.000 -ATOM 5356 OH2 TIP3 6867 8.346 13.096 -3.718 -0.834 1.520 -ATOM 5357 H1 TIP3 6867 8.406 14.067 -3.550 0.417 1.000 -ATOM 5358 H2 TIP3 6867 7.374 12.973 -3.851 0.417 1.000 -ATOM 5359 OH2 TIP3 6877 11.790 10.649 8.873 -0.834 1.520 -ATOM 5360 H1 TIP3 6877 11.748 11.503 9.268 0.417 1.000 -ATOM 5361 H2 TIP3 6877 12.731 10.397 8.971 0.417 1.000 -ATOM 5362 OH2 TIP3 6917 17.309 11.938 10.025 -0.834 1.520 -ATOM 5363 H1 TIP3 6917 17.406 11.032 9.653 0.417 1.000 -ATOM 5364 H2 TIP3 6917 17.874 12.461 9.446 0.417 1.000 -ATOM 5365 OH2 TIP3 6975 16.688 -15.729 -2.742 -0.834 1.520 -ATOM 5366 H1 TIP3 6975 16.893 -15.870 -3.654 0.417 1.000 -ATOM 5367 H2 TIP3 6975 15.924 -16.267 -2.556 0.417 1.000 -ATOM 5368 OH2 TIP3 6979 18.530 8.226 15.245 -0.834 1.520 -ATOM 5369 H1 TIP3 6979 19.218 8.419 15.889 0.417 1.000 -ATOM 5370 H2 TIP3 6979 18.179 7.357 15.475 0.417 1.000 -ATOM 5371 OH2 TIP3 7017 16.067 -18.799 -17.653 -0.834 1.520 -ATOM 5372 H1 TIP3 7017 15.519 -18.640 -18.474 0.417 1.000 -ATOM 5373 H2 TIP3 7017 16.916 -18.501 -17.937 0.417 1.000 -ATOM 5374 OH2 TIP3 7021 19.529 -9.441 1.670 -0.834 1.520 -ATOM 5375 H1 TIP3 7021 19.411 -8.627 2.121 0.417 1.000 -ATOM 5376 H2 TIP3 7021 19.816 -9.096 0.723 0.417 1.000 -ATOM 5377 OH2 TIP3 7027 -19.283 -18.618 15.598 -0.834 1.520 -ATOM 5378 H1 TIP3 7027 -18.453 -19.138 15.690 0.417 1.000 -ATOM 5379 H2 TIP3 7027 -19.453 -18.376 16.484 0.417 1.000 -ATOM 5380 OH2 TIP3 7041 17.833 -8.618 -6.082 -0.834 1.520 -ATOM 5381 H1 TIP3 7041 17.472 -9.335 -6.648 0.417 1.000 -ATOM 5382 H2 TIP3 7041 18.290 -9.112 -5.415 0.417 1.000 -ATOM 5383 OH2 TIP3 7052 14.779 -4.828 -11.945 -0.834 1.520 -ATOM 5384 H1 TIP3 7052 15.198 -5.207 -11.212 0.417 1.000 -ATOM 5385 H2 TIP3 7052 14.457 -5.617 -12.426 0.417 1.000 -ATOM 5386 OH2 TIP3 7058 7.696 -1.366 -5.657 -0.834 1.520 -ATOM 5387 H1 TIP3 7058 6.941 -0.984 -5.209 0.417 1.000 -ATOM 5388 H2 TIP3 7058 8.424 -0.986 -5.075 0.417 1.000 -ATOM 5389 OH2 TIP3 7062 18.062 -4.590 0.332 -0.834 1.520 -ATOM 5390 H1 TIP3 7062 17.339 -4.860 0.883 0.417 1.000 -ATOM 5391 H2 TIP3 7062 17.656 -3.997 -0.369 0.417 1.000 -ATOM 5392 OH2 TIP3 7068 13.480 -11.792 12.114 -0.834 1.520 -ATOM 5393 H1 TIP3 7068 13.573 -10.949 11.593 0.417 1.000 -ATOM 5394 H2 TIP3 7068 14.407 -12.124 12.157 0.417 1.000 -ATOM 5395 OH2 TIP3 7078 13.653 1.374 -15.511 -0.834 1.520 -ATOM 5396 H1 TIP3 7078 13.702 0.838 -16.298 0.417 1.000 -ATOM 5397 H2 TIP3 7078 12.888 1.034 -15.044 0.417 1.000 -ATOM 5398 OH2 TIP3 7121 17.815 -13.426 -6.045 -0.834 1.520 -ATOM 5399 H1 TIP3 7121 16.975 -12.898 -6.101 0.417 1.000 -ATOM 5400 H2 TIP3 7121 18.523 -12.863 -5.927 0.417 1.000 -ATOM 5401 OH2 TIP3 7130 17.965 -11.088 18.026 -0.834 1.520 -ATOM 5402 H1 TIP3 7130 17.727 -10.479 17.293 0.417 1.000 -ATOM 5403 H2 TIP3 7130 18.030 -10.567 18.768 0.417 1.000 -ATOM 5404 OH2 TIP3 7133 -16.425 -2.423 15.420 -0.834 1.520 -ATOM 5405 H1 TIP3 7133 -15.472 -2.401 15.232 0.417 1.000 -ATOM 5406 H2 TIP3 7133 -16.705 -3.257 15.025 0.417 1.000 -ATOM 5407 OH2 TIP3 7141 -3.702 2.765 -5.053 -0.834 1.520 -ATOM 5408 H1 TIP3 7141 -3.906 2.020 -5.582 0.417 1.000 -ATOM 5409 H2 TIP3 7141 -2.737 2.705 -4.973 0.417 1.000 -ATOM 5410 OH2 TIP3 7147 16.166 6.356 -13.323 -0.834 1.520 -ATOM 5411 H1 TIP3 7147 15.328 6.530 -12.894 0.417 1.000 -ATOM 5412 H2 TIP3 7147 16.749 7.118 -12.998 0.417 1.000 -ATOM 5413 OH2 TIP3 7170 17.428 2.797 14.373 -0.834 1.520 -ATOM 5414 H1 TIP3 7170 17.633 3.370 15.112 0.417 1.000 -ATOM 5415 H2 TIP3 7170 16.520 2.918 14.215 0.417 1.000 -ATOM 5416 OH2 TIP3 7174 19.220 -12.987 15.804 -0.834 1.520 -ATOM 5417 H1 TIP3 7174 18.677 -12.417 16.388 0.417 1.000 -ATOM 5418 H2 TIP3 7174 19.322 -13.757 16.344 0.417 1.000 -ATOM 5419 OH2 TIP3 7192 15.673 9.757 5.294 -0.834 1.520 -ATOM 5420 H1 TIP3 7192 16.044 10.251 4.538 0.417 1.000 -ATOM 5421 H2 TIP3 7192 15.922 10.353 6.042 0.417 1.000 -ATOM 5422 OH2 TIP3 7222 -13.745 -10.198 -19.636 -0.834 1.520 -ATOM 5423 H1 TIP3 7222 -13.046 -10.879 -19.329 0.417 1.000 -ATOM 5424 H2 TIP3 7222 -14.541 -10.736 -19.804 0.417 1.000 -ATOM 5425 OH2 TIP3 7252 10.691 -0.088 5.720 -0.834 1.520 -ATOM 5426 H1 TIP3 7252 11.071 0.322 4.919 0.417 1.000 -ATOM 5427 H2 TIP3 7252 11.454 -0.476 6.131 0.417 1.000 -ATOM 5428 OH2 TIP3 7295 -19.149 2.646 6.279 -0.834 1.520 -ATOM 5429 H1 TIP3 7295 -18.913 1.836 5.835 0.417 1.000 -ATOM 5430 H2 TIP3 7295 -19.759 2.368 6.986 0.417 1.000 -ATOM 5431 OH2 TIP3 7335 18.795 19.215 -9.145 -0.834 1.520 -ATOM 5432 H1 TIP3 7335 19.201 19.840 -9.737 0.417 1.000 -ATOM 5433 H2 TIP3 7335 18.593 18.469 -9.736 0.417 1.000 -ATOM 5434 OH2 TIP3 7352 17.693 13.942 -7.250 -0.834 1.520 -ATOM 5435 H1 TIP3 7352 18.336 14.171 -6.505 0.417 1.000 -ATOM 5436 H2 TIP3 7352 16.961 14.518 -7.055 0.417 1.000 -ATOM 5437 OH2 TIP3 7374 14.785 14.606 -11.307 -0.834 1.520 -ATOM 5438 H1 TIP3 7374 14.825 14.231 -12.199 0.417 1.000 -ATOM 5439 H2 TIP3 7374 13.990 14.217 -10.910 0.417 1.000 -ATOM 5440 OH2 TIP3 7395 18.647 16.816 -10.714 -0.834 1.520 -ATOM 5441 H1 TIP3 7395 19.199 16.093 -11.008 0.417 1.000 -ATOM 5442 H2 TIP3 7395 17.955 16.462 -10.194 0.417 1.000 -ATOM 5443 OH2 TIP3 7437 16.637 -16.038 -6.827 -0.834 1.520 -ATOM 5444 H1 TIP3 7437 17.082 -15.250 -6.454 0.417 1.000 -ATOM 5445 H2 TIP3 7437 17.123 -16.088 -7.717 0.417 1.000 -ATOM 5446 OH2 TIP3 7479 -13.860 -16.331 -16.296 -0.834 1.520 -ATOM 5447 H1 TIP3 7479 -13.268 -16.163 -17.051 0.417 1.000 -ATOM 5448 H2 TIP3 7479 -13.193 -16.592 -15.622 0.417 1.000 -ATOM 5449 OH2 TIP3 7501 13.476 -15.160 -11.139 -0.834 1.520 -ATOM 5450 H1 TIP3 7501 13.896 -15.340 -10.325 0.417 1.000 -ATOM 5451 H2 TIP3 7501 13.229 -16.074 -11.329 0.417 1.000 -ATOM 5452 OH2 TIP3 7510 -14.850 -8.315 -8.006 -0.834 1.520 -ATOM 5453 H1 TIP3 7510 -14.688 -7.359 -7.980 0.417 1.000 -ATOM 5454 H2 TIP3 7510 -14.386 -8.595 -7.222 0.417 1.000 -ATOM 5455 OH2 TIP3 7525 12.999 -10.993 1.240 -0.834 1.520 -ATOM 5456 H1 TIP3 7525 12.490 -11.055 0.384 0.417 1.000 -ATOM 5457 H2 TIP3 7525 12.823 -11.900 1.616 0.417 1.000 -ATOM 5458 OH2 TIP3 7532 -18.993 -11.593 17.837 -0.834 1.520 -ATOM 5459 H1 TIP3 7532 -18.402 -12.206 17.369 0.417 1.000 -ATOM 5460 H2 TIP3 7532 -18.847 -11.855 18.765 0.417 1.000 -ATOM 5461 OH2 TIP3 7563 -14.804 -0.696 -4.685 -0.834 1.520 -ATOM 5462 H1 TIP3 7563 -14.773 0.213 -5.058 0.417 1.000 -ATOM 5463 H2 TIP3 7563 -14.748 -1.280 -5.431 0.417 1.000 -ATOM 5464 OH2 TIP3 7567 16.186 -4.390 -2.041 -0.834 1.520 -ATOM 5465 H1 TIP3 7567 15.536 -3.812 -2.525 0.417 1.000 -ATOM 5466 H2 TIP3 7567 15.630 -4.621 -1.255 0.417 1.000 -ATOM 5467 OH2 TIP3 7568 -16.319 -12.656 3.826 -0.834 1.520 -ATOM 5468 H1 TIP3 7568 -16.420 -12.533 4.764 0.417 1.000 -ATOM 5469 H2 TIP3 7568 -16.369 -11.739 3.500 0.417 1.000 -ATOM 5470 OH2 TIP3 7592 -15.391 4.656 13.926 -0.834 1.520 -ATOM 5471 H1 TIP3 7592 -15.711 5.189 14.710 0.417 1.000 -ATOM 5472 H2 TIP3 7592 -15.687 3.763 14.198 0.417 1.000 -ATOM 5473 OH2 TIP3 7603 -15.354 16.705 -14.461 -0.834 1.520 -ATOM 5474 H1 TIP3 7603 -16.194 17.258 -14.588 0.417 1.000 -ATOM 5475 H2 TIP3 7603 -14.904 16.891 -15.260 0.417 1.000 -ATOM 5476 OH2 TIP3 7605 -13.313 4.853 -9.652 -0.834 1.520 -ATOM 5477 H1 TIP3 7605 -12.412 4.883 -9.271 0.417 1.000 -ATOM 5478 H2 TIP3 7605 -13.156 4.632 -10.581 0.417 1.000 -ATOM 5479 OH2 TIP3 7612 19.974 -5.947 -0.977 -0.834 1.520 -ATOM 5480 H1 TIP3 7612 19.316 -6.427 -1.528 0.417 1.000 -ATOM 5481 H2 TIP3 7612 19.402 -5.571 -0.386 0.417 1.000 -ATOM 5482 OH2 TIP3 7630 -17.821 -6.536 -12.823 -0.834 1.520 -ATOM 5483 H1 TIP3 7630 -17.281 -5.939 -12.263 0.417 1.000 -ATOM 5484 H2 TIP3 7630 -18.360 -6.979 -12.157 0.417 1.000 -ATOM 5485 OH2 TIP3 7631 -18.074 1.439 -2.973 -0.834 1.520 -ATOM 5486 H1 TIP3 7631 -17.328 1.817 -3.455 0.417 1.000 -ATOM 5487 H2 TIP3 7631 -17.602 0.912 -2.338 0.417 1.000 -ATOM 5488 OH2 TIP3 7634 -16.686 -8.049 13.268 -0.834 1.520 -ATOM 5489 H1 TIP3 7634 -16.819 -7.663 12.373 0.417 1.000 -ATOM 5490 H2 TIP3 7634 -15.934 -8.600 13.009 0.417 1.000 -ATOM 5491 OH2 TIP3 7643 15.653 5.857 -16.046 -0.834 1.520 -ATOM 5492 H1 TIP3 7643 15.956 6.183 -15.187 0.417 1.000 -ATOM 5493 H2 TIP3 7643 16.136 6.461 -16.642 0.417 1.000 -ATOM 5494 OH2 TIP3 7647 -18.693 5.327 -9.867 -0.834 1.520 -ATOM 5495 H1 TIP3 7647 -17.748 5.463 -9.923 0.417 1.000 -ATOM 5496 H2 TIP3 7647 -18.878 4.555 -10.528 0.417 1.000 -ATOM 5497 OH2 TIP3 7650 -15.902 7.516 -15.559 -0.834 1.520 -ATOM 5498 H1 TIP3 7650 -16.131 6.827 -14.935 0.417 1.000 -ATOM 5499 H2 TIP3 7650 -14.916 7.574 -15.407 0.417 1.000 -ATOM 5500 OH2 TIP3 7661 -16.960 -4.174 17.777 -0.834 1.520 -ATOM 5501 H1 TIP3 7661 -17.448 -3.363 17.476 0.417 1.000 -ATOM 5502 H2 TIP3 7661 -17.726 -4.788 18.160 0.417 1.000 -ATOM 5503 OH2 TIP3 7668 12.031 6.248 -6.222 -0.834 1.520 -ATOM 5504 H1 TIP3 7668 11.580 7.134 -6.055 0.417 1.000 -ATOM 5505 H2 TIP3 7668 12.425 6.036 -5.354 0.417 1.000 -ATOM 5506 OH2 TIP3 7676 -12.967 -8.515 11.967 -0.834 1.520 -ATOM 5507 H1 TIP3 7676 -12.071 -8.669 12.292 0.417 1.000 -ATOM 5508 H2 TIP3 7676 -13.455 -9.240 12.326 0.417 1.000 -ATOM 5509 OH2 TIP3 7690 13.776 4.452 -14.397 -0.834 1.520 -ATOM 5510 H1 TIP3 7690 14.121 4.657 -15.315 0.417 1.000 -ATOM 5511 H2 TIP3 7690 13.743 5.307 -13.945 0.417 1.000 -ATOM 5512 OH2 TIP3 7707 -19.312 -17.219 -12.886 -0.834 1.520 -ATOM 5513 H1 TIP3 7707 -18.512 -16.810 -13.297 0.417 1.000 -ATOM 5514 H2 TIP3 7707 -19.947 -16.571 -13.070 0.417 1.000 -ATOM 5515 OH2 TIP3 7715 -17.093 18.455 2.007 -0.834 1.520 -ATOM 5516 H1 TIP3 7715 -17.451 18.286 2.898 0.417 1.000 -ATOM 5517 H2 TIP3 7715 -16.597 19.349 2.100 0.417 1.000 -ATOM 5518 OH2 TIP3 7716 -12.598 8.660 -9.639 -0.834 1.520 -ATOM 5519 H1 TIP3 7716 -13.391 9.153 -9.466 0.417 1.000 -ATOM 5520 H2 TIP3 7716 -12.363 8.912 -10.514 0.417 1.000 -ATOM 5521 OH2 TIP3 7739 -15.403 3.385 2.279 -0.834 1.520 -ATOM 5522 H1 TIP3 7739 -16.085 3.520 1.604 0.417 1.000 -ATOM 5523 H2 TIP3 7739 -15.588 4.142 2.857 0.417 1.000 -ATOM 5524 OH2 TIP3 7748 -18.613 12.752 -18.148 -0.834 1.520 -ATOM 5525 H1 TIP3 7748 -17.761 12.757 -18.474 0.417 1.000 -ATOM 5526 H2 TIP3 7748 -18.484 12.728 -17.200 0.417 1.000 -ATOM 5527 OH2 TIP3 7860 17.007 -4.910 8.251 -0.834 1.520 -ATOM 5528 H1 TIP3 7860 16.296 -5.590 8.477 0.417 1.000 -ATOM 5529 H2 TIP3 7860 16.957 -4.922 7.278 0.417 1.000 -ATOM 5530 OH2 TIP3 7904 19.140 -19.452 9.228 -0.834 1.520 -ATOM 5531 H1 TIP3 7904 18.639 -19.941 8.527 0.417 1.000 -ATOM 5532 H2 TIP3 7904 19.818 -18.903 8.770 0.417 1.000 -ATOM 5533 OH2 TIP3 7945 -10.765 -14.413 3.329 -0.834 1.520 -ATOM 5534 H1 TIP3 7945 -10.542 -13.865 4.099 0.417 1.000 -ATOM 5535 H2 TIP3 7945 -11.515 -13.869 2.972 0.417 1.000 -ATOM 5536 OH2 TIP3 7951 -16.555 -16.133 18.949 -0.834 1.520 -ATOM 5537 H1 TIP3 7951 -16.199 -16.682 18.209 0.417 1.000 -ATOM 5538 H2 TIP3 7951 -16.814 -16.782 19.626 0.417 1.000 -ATOM 5539 OH2 TIP3 7964 17.329 -0.581 -5.197 -0.834 1.520 -ATOM 5540 H1 TIP3 7964 18.207 -0.712 -4.713 0.417 1.000 -ATOM 5541 H2 TIP3 7964 17.005 -1.481 -5.308 0.417 1.000 -ATOM 5542 OH2 TIP3 7985 -19.551 -12.634 -10.860 -0.834 1.520 -ATOM 5543 H1 TIP3 7985 -19.798 -13.521 -10.905 0.417 1.000 -ATOM 5544 H2 TIP3 7985 -19.204 -12.437 -11.744 0.417 1.000 -ATOM 5545 OH2 TIP3 8014 -16.496 1.790 14.010 -0.834 1.520 -ATOM 5546 H1 TIP3 8014 -15.973 1.570 13.237 0.417 1.000 -ATOM 5547 H2 TIP3 8014 -17.156 2.371 13.635 0.417 1.000 -ATOM 5548 OH2 TIP3 8023 -19.087 -7.680 -19.063 -0.834 1.520 -ATOM 5549 H1 TIP3 8023 -19.377 -8.300 -19.732 0.417 1.000 -ATOM 5550 H2 TIP3 8023 -18.710 -8.216 -18.372 0.417 1.000 -ATOM 5551 OH2 TIP3 8024 -15.891 5.989 -9.851 -0.834 1.520 -ATOM 5552 H1 TIP3 8024 -14.997 5.645 -9.650 0.417 1.000 -ATOM 5553 H2 TIP3 8024 -15.683 6.768 -10.417 0.417 1.000 -ATOM 5554 OH2 TIP3 8029 -12.111 -3.003 -5.640 -0.834 1.520 -ATOM 5555 H1 TIP3 8029 -12.065 -2.819 -6.642 0.417 1.000 -ATOM 5556 H2 TIP3 8029 -11.398 -2.438 -5.266 0.417 1.000 -ATOM 5557 OH2 TIP3 8046 -16.770 2.882 -8.264 -0.834 1.520 -ATOM 5558 H1 TIP3 8046 -16.399 1.950 -8.330 0.417 1.000 -ATOM 5559 H2 TIP3 8046 -17.139 2.992 -9.147 0.417 1.000 -ATOM 5560 OH2 TIP3 8055 -13.838 -2.289 15.087 -0.834 1.520 -ATOM 5561 H1 TIP3 8055 -13.356 -2.334 15.913 0.417 1.000 -ATOM 5562 H2 TIP3 8055 -13.434 -1.507 14.651 0.417 1.000 -ATOM 5563 OH2 TIP3 8064 -15.380 2.335 -12.870 -0.834 1.520 -ATOM 5564 H1 TIP3 8064 -16.246 2.591 -12.410 0.417 1.000 -ATOM 5565 H2 TIP3 8064 -14.822 2.349 -12.093 0.417 1.000 -ATOM 5566 OH2 TIP3 8067 -18.478 9.688 -5.200 -0.834 1.520 -ATOM 5567 H1 TIP3 8067 -18.476 10.417 -5.842 0.417 1.000 -ATOM 5568 H2 TIP3 8067 -17.570 9.403 -4.980 0.417 1.000 -ATOM 5569 OH2 TIP3 8068 -18.876 2.962 -6.486 -0.834 1.520 -ATOM 5570 H1 TIP3 8068 -18.163 2.901 -7.140 0.417 1.000 -ATOM 5571 H2 TIP3 8068 -19.315 3.799 -6.690 0.417 1.000 -ATOM 5572 OH2 TIP3 8070 13.465 -12.498 -12.880 -0.834 1.520 -ATOM 5573 H1 TIP3 8070 13.340 -12.871 -13.808 0.417 1.000 -ATOM 5574 H2 TIP3 8070 13.740 -13.267 -12.324 0.417 1.000 -ATOM 5575 OH2 TIP3 8073 16.526 -2.619 9.510 -0.834 1.520 -ATOM 5576 H1 TIP3 8073 16.955 -3.347 9.033 0.417 1.000 -ATOM 5577 H2 TIP3 8073 16.593 -2.909 10.428 0.417 1.000 -ATOM 5578 OH2 TIP3 8076 -12.634 4.266 8.891 -0.834 1.520 -ATOM 5579 H1 TIP3 8076 -13.480 4.258 9.415 0.417 1.000 -ATOM 5580 H2 TIP3 8076 -12.923 3.998 7.984 0.417 1.000 -ATOM 5581 OH2 TIP3 8094 -11.265 4.622 18.928 -0.834 1.520 -ATOM 5582 H1 TIP3 8094 -10.531 4.022 18.715 0.417 1.000 -ATOM 5583 H2 TIP3 8094 -11.728 4.616 18.104 0.417 1.000 -ATOM 5584 OH2 TIP3 8133 -18.962 9.805 4.340 -0.834 1.520 -ATOM 5585 H1 TIP3 8133 -18.822 8.861 4.127 0.417 1.000 -ATOM 5586 H2 TIP3 8133 -19.353 9.734 5.231 0.417 1.000 -ATOM 5587 OH2 TIP3 8136 -18.586 11.430 -7.698 -0.834 1.520 -ATOM 5588 H1 TIP3 8136 -17.925 10.965 -8.249 0.417 1.000 -ATOM 5589 H2 TIP3 8136 -18.258 12.354 -7.659 0.417 1.000 -ATOM 5590 OH2 TIP3 8174 -17.237 6.896 -7.696 -0.834 1.520 -ATOM 5591 H1 TIP3 8174 -16.599 6.364 -8.255 0.417 1.000 -ATOM 5592 H2 TIP3 8174 -17.249 7.725 -8.141 0.417 1.000 -ATOM 5593 OH2 TIP3 8176 -15.886 13.989 8.277 -0.834 1.520 -ATOM 5594 H1 TIP3 8176 -15.618 14.939 8.099 0.417 1.000 -ATOM 5595 H2 TIP3 8176 -15.653 13.906 9.206 0.417 1.000 -ATOM 5596 OH2 TIP3 8185 17.580 6.104 16.715 -0.834 1.520 -ATOM 5597 H1 TIP3 8185 16.808 5.636 16.356 0.417 1.000 -ATOM 5598 H2 TIP3 8185 17.695 5.614 17.578 0.417 1.000 -ATOM 5599 OH2 TIP3 8193 -18.204 15.376 -13.042 -0.834 1.520 -ATOM 5600 H1 TIP3 8193 -19.138 15.378 -13.378 0.417 1.000 -ATOM 5601 H2 TIP3 8193 -17.818 16.170 -13.647 0.417 1.000 -ATOM 5602 OH2 TIP3 8206 -9.087 6.333 19.362 -0.834 1.520 -ATOM 5603 H1 TIP3 8206 -9.778 5.705 19.171 0.417 1.000 -ATOM 5604 H2 TIP3 8206 -8.496 5.710 19.872 0.417 1.000 -ATOM 5605 OH2 TIP3 8222 -15.523 19.384 14.416 -0.834 1.520 -ATOM 5606 H1 TIP3 8222 -15.396 18.706 13.741 0.417 1.000 -ATOM 5607 H2 TIP3 8222 -15.890 18.813 15.147 0.417 1.000 -ATOM 5608 OH2 TIP3 8242 -5.562 18.821 0.722 -0.834 1.520 -ATOM 5609 H1 TIP3 8242 -4.820 18.250 0.442 0.417 1.000 -ATOM 5610 H2 TIP3 8242 -6.254 18.198 1.081 0.417 1.000 -ATOM 5611 OH2 TIP3 8263 -16.761 16.744 8.122 -0.834 1.520 -ATOM 5612 H1 TIP3 8263 -17.393 16.352 8.722 0.417 1.000 -ATOM 5613 H2 TIP3 8263 -17.291 17.003 7.327 0.417 1.000 -ATOM 5614 OH2 TIP3 8265 -17.940 14.822 17.197 -0.834 1.520 -ATOM 5615 H1 TIP3 8265 -18.139 15.621 17.663 0.417 1.000 -ATOM 5616 H2 TIP3 8265 -16.981 14.885 17.279 0.417 1.000 -ATOM 5617 OH2 TIP3 8326 18.000 -15.488 7.686 -0.834 1.520 -ATOM 5618 H1 TIP3 8326 17.605 -16.367 7.423 0.417 1.000 -ATOM 5619 H2 TIP3 8326 17.181 -14.998 7.900 0.417 1.000 -ATOM 5620 OH2 TIP3 8364 -11.439 -9.076 -3.119 -0.834 1.520 -ATOM 5621 H1 TIP3 8364 -12.084 -8.812 -3.798 0.417 1.000 -ATOM 5622 H2 TIP3 8364 -11.691 -9.992 -2.877 0.417 1.000 -ATOM 5623 OH2 TIP3 8370 -3.988 -15.932 4.241 -0.834 1.520 -ATOM 5624 H1 TIP3 8370 -3.379 -16.214 3.521 0.417 1.000 -ATOM 5625 H2 TIP3 8370 -3.445 -15.992 5.013 0.417 1.000 -ATOM 5626 OH2 TIP3 8394 -11.010 -12.369 16.752 -0.834 1.520 -ATOM 5627 H1 TIP3 8394 -10.423 -11.699 17.070 0.417 1.000 -ATOM 5628 H2 TIP3 8394 -11.840 -12.279 17.323 0.417 1.000 -ATOM 5629 OH2 TIP3 8403 -15.249 -0.991 -17.186 -0.834 1.520 -ATOM 5630 H1 TIP3 8403 -15.246 -1.022 -16.240 0.417 1.000 -ATOM 5631 H2 TIP3 8403 -14.700 -1.694 -17.388 0.417 1.000 -ATOM 5632 OH2 TIP3 8408 -19.409 -15.347 5.328 -0.834 1.520 -ATOM 5633 H1 TIP3 8408 -19.253 -15.507 4.381 0.417 1.000 -ATOM 5634 H2 TIP3 8408 -18.744 -15.897 5.785 0.417 1.000 -ATOM 5635 OH2 TIP3 8409 13.953 -13.595 2.441 -0.834 1.520 -ATOM 5636 H1 TIP3 8409 14.304 -14.391 1.938 0.417 1.000 -ATOM 5637 H2 TIP3 8409 14.567 -12.876 2.197 0.417 1.000 -ATOM 5638 OH2 TIP3 8411 -16.594 -19.883 -5.507 -0.834 1.520 -ATOM 5639 H1 TIP3 8411 -16.652 -19.756 -6.454 0.417 1.000 -ATOM 5640 H2 TIP3 8411 -15.622 -19.810 -5.276 0.417 1.000 -ATOM 5641 OH2 TIP3 8412 -11.831 -17.863 17.598 -0.834 1.520 -ATOM 5642 H1 TIP3 8412 -11.739 -16.914 17.790 0.417 1.000 -ATOM 5643 H2 TIP3 8412 -12.095 -17.832 16.631 0.417 1.000 -ATOM 5644 OH2 TIP3 8425 -17.722 -16.078 -18.800 -0.834 1.520 -ATOM 5645 H1 TIP3 8425 -17.299 -16.608 -19.440 0.417 1.000 -ATOM 5646 H2 TIP3 8425 -18.444 -16.594 -18.585 0.417 1.000 -ATOM 5647 OH2 TIP3 8441 -13.754 1.216 -6.436 -0.834 1.520 -ATOM 5648 H1 TIP3 8441 -12.873 1.533 -6.206 0.417 1.000 -ATOM 5649 H2 TIP3 8441 -14.198 2.075 -6.545 0.417 1.000 -ATOM 5650 OH2 TIP3 8448 -19.584 -0.598 6.409 -0.834 1.520 -ATOM 5651 H1 TIP3 8448 -18.628 -0.288 6.262 0.417 1.000 -ATOM 5652 H2 TIP3 8448 -19.472 -1.461 6.599 0.417 1.000 -ATOM 5653 OH2 TIP3 8457 -10.564 -4.268 12.393 -0.834 1.520 -ATOM 5654 H1 TIP3 8457 -11.188 -4.698 12.981 0.417 1.000 -ATOM 5655 H2 TIP3 8457 -9.713 -4.285 12.899 0.417 1.000 -ATOM 5656 OH2 TIP3 8462 18.064 0.344 -9.343 -0.834 1.520 -ATOM 5657 H1 TIP3 8462 18.211 -0.518 -8.908 0.417 1.000 -ATOM 5658 H2 TIP3 8462 17.319 0.197 -10.027 0.417 1.000 -ATOM 5659 OH2 TIP3 8466 -14.652 -4.541 -14.012 -0.834 1.520 -ATOM 5660 H1 TIP3 8466 -15.058 -3.789 -13.535 0.417 1.000 -ATOM 5661 H2 TIP3 8466 -15.036 -5.321 -13.578 0.417 1.000 -ATOM 5662 OH2 TIP3 8468 -18.688 -6.423 -6.171 -0.834 1.520 -ATOM 5663 H1 TIP3 8468 -18.406 -6.650 -7.037 0.417 1.000 -ATOM 5664 H2 TIP3 8468 -19.095 -5.557 -6.366 0.417 1.000 -ATOM 5665 OH2 TIP3 8491 -9.605 1.375 2.397 -0.834 1.520 -ATOM 5666 H1 TIP3 8491 -9.100 0.680 2.794 0.417 1.000 -ATOM 5667 H2 TIP3 8491 -9.318 2.154 2.828 0.417 1.000 -ATOM 5668 OH2 TIP3 8492 -18.407 3.215 12.879 -0.834 1.520 -ATOM 5669 H1 TIP3 8492 -19.142 3.389 13.452 0.417 1.000 -ATOM 5670 H2 TIP3 8492 -18.801 3.016 12.034 0.417 1.000 -ATOM 5671 OH2 TIP3 8509 19.685 -15.552 -13.837 -0.834 1.520 -ATOM 5672 H1 TIP3 8509 19.380 -15.628 -12.919 0.417 1.000 -ATOM 5673 H2 TIP3 8509 19.772 -16.432 -14.133 0.417 1.000 -ATOM 5674 OH2 TIP3 8528 -13.987 -3.915 -16.674 -0.834 1.520 -ATOM 5675 H1 TIP3 8528 -13.984 -3.965 -15.708 0.417 1.000 -ATOM 5676 H2 TIP3 8528 -14.751 -4.484 -16.954 0.417 1.000 -ATOM 5677 OH2 TIP3 8534 -17.816 2.110 -10.500 -0.834 1.520 -ATOM 5678 H1 TIP3 8534 -18.087 1.297 -10.002 0.417 1.000 -ATOM 5679 H2 TIP3 8534 -18.611 2.233 -11.036 0.417 1.000 -ATOM 5680 OH2 TIP3 8536 -16.793 -6.021 -2.228 -0.834 1.520 -ATOM 5681 H1 TIP3 8536 -16.053 -5.918 -1.627 0.417 1.000 -ATOM 5682 H2 TIP3 8536 -17.415 -5.315 -2.079 0.417 1.000 -ATOM 5683 OH2 TIP3 8548 -6.361 -3.902 -14.731 -0.834 1.520 -ATOM 5684 H1 TIP3 8548 -5.901 -4.606 -15.128 0.417 1.000 -ATOM 5685 H2 TIP3 8548 -7.245 -4.276 -14.657 0.417 1.000 -ATOM 5686 OH2 TIP3 8549 -15.402 8.408 -1.681 -0.834 1.520 -ATOM 5687 H1 TIP3 8549 -15.331 8.317 -2.645 0.417 1.000 -ATOM 5688 H2 TIP3 8549 -14.610 7.828 -1.358 0.417 1.000 -ATOM 5689 OH2 TIP3 8551 -13.816 14.140 -15.946 -0.834 1.520 -ATOM 5690 H1 TIP3 8551 -14.257 14.819 -16.528 0.417 1.000 -ATOM 5691 H2 TIP3 8551 -14.494 13.825 -15.380 0.417 1.000 -ATOM 5692 OH2 TIP3 8555 -13.816 1.625 4.372 -0.834 1.520 -ATOM 5693 H1 TIP3 8555 -14.369 1.179 3.798 0.417 1.000 -ATOM 5694 H2 TIP3 8555 -14.042 1.386 5.308 0.417 1.000 -ATOM 5695 OH2 TIP3 8556 -18.019 -3.365 -1.337 -0.834 1.520 -ATOM 5696 H1 TIP3 8556 -18.079 -2.819 -0.471 0.417 1.000 -ATOM 5697 H2 TIP3 8556 -17.344 -2.883 -1.844 0.417 1.000 -ATOM 5698 OH2 TIP3 8558 -17.987 5.789 -5.125 -0.834 1.520 -ATOM 5699 H1 TIP3 8558 -18.922 5.523 -4.994 0.417 1.000 -ATOM 5700 H2 TIP3 8558 -17.882 6.044 -6.077 0.417 1.000 -ATOM 5701 OH2 TIP3 8566 -18.671 3.741 17.937 -0.834 1.520 -ATOM 5702 H1 TIP3 8566 -18.831 3.178 17.166 0.417 1.000 -ATOM 5703 H2 TIP3 8566 -19.277 4.496 17.762 0.417 1.000 -ATOM 5704 OH2 TIP3 8574 -15.893 8.282 -4.761 -0.834 1.520 -ATOM 5705 H1 TIP3 8574 -15.023 8.023 -5.218 0.417 1.000 -ATOM 5706 H2 TIP3 8574 -16.446 7.530 -4.982 0.417 1.000 -ATOM 5707 OH2 TIP3 8576 -9.957 16.852 -6.288 -0.834 1.520 -ATOM 5708 H1 TIP3 8576 -9.172 17.292 -5.846 0.417 1.000 -ATOM 5709 H2 TIP3 8576 -10.671 17.362 -6.055 0.417 1.000 -ATOM 5710 OH2 TIP3 8578 -13.604 18.881 5.021 -0.834 1.520 -ATOM 5711 H1 TIP3 8578 -13.286 18.520 4.184 0.417 1.000 -ATOM 5712 H2 TIP3 8578 -13.507 18.150 5.663 0.417 1.000 -ATOM 5713 OH2 TIP3 8579 -13.792 10.748 6.763 -0.834 1.520 -ATOM 5714 H1 TIP3 8579 -13.747 11.642 6.366 0.417 1.000 -ATOM 5715 H2 TIP3 8579 -14.788 10.523 6.975 0.417 1.000 -ATOM 5716 OH2 TIP3 8600 -14.632 7.922 11.937 -0.834 1.520 -ATOM 5717 H1 TIP3 8600 -15.523 7.601 12.188 0.417 1.000 -ATOM 5718 H2 TIP3 8600 -14.616 7.715 11.037 0.417 1.000 -ATOM 5719 OH2 TIP3 8613 -11.883 13.380 -17.732 -0.834 1.520 -ATOM 5720 H1 TIP3 8613 -12.415 13.594 -17.008 0.417 1.000 -ATOM 5721 H2 TIP3 8613 -11.143 14.005 -17.527 0.417 1.000 -ATOM 5722 OH2 TIP3 8617 -11.864 17.527 -12.535 -0.834 1.520 -ATOM 5723 H1 TIP3 8617 -12.147 18.042 -11.719 0.417 1.000 -ATOM 5724 H2 TIP3 8617 -11.347 16.807 -12.108 0.417 1.000 -ATOM 5725 OH2 TIP3 8642 -15.194 15.050 16.989 -0.834 1.520 -ATOM 5726 H1 TIP3 8642 -14.868 14.161 16.565 0.417 1.000 -ATOM 5727 H2 TIP3 8642 -14.869 15.693 16.374 0.417 1.000 -ATOM 5728 OH2 TIP3 8645 -19.641 9.801 6.958 -0.834 1.520 -ATOM 5729 H1 TIP3 8645 -19.967 9.183 7.647 0.417 1.000 -ATOM 5730 H2 TIP3 8645 -19.944 10.709 7.128 0.417 1.000 -ATOM 5731 OH2 TIP3 8653 -18.572 19.027 -17.845 -0.834 1.520 -ATOM 5732 H1 TIP3 8653 -18.507 19.600 -16.989 0.417 1.000 -ATOM 5733 H2 TIP3 8653 -18.462 18.186 -17.492 0.417 1.000 -ATOM 5734 OH2 TIP3 8658 -17.830 18.392 16.274 -0.834 1.520 -ATOM 5735 H1 TIP3 8658 -18.417 17.888 15.662 0.417 1.000 -ATOM 5736 H2 TIP3 8658 -18.289 19.231 16.405 0.417 1.000 -ATOM 5737 OH2 TIP3 8679 -17.787 17.935 -14.357 -0.834 1.520 -ATOM 5738 H1 TIP3 8679 -17.845 18.521 -13.599 0.417 1.000 -ATOM 5739 H2 TIP3 8679 -17.927 18.558 -15.075 0.417 1.000 -ATOM 5740 OH2 TIP3 8683 -19.503 15.881 1.658 -0.834 1.520 -ATOM 5741 H1 TIP3 8683 -19.764 15.928 0.778 0.417 1.000 -ATOM 5742 H2 TIP3 8683 -18.803 16.514 1.645 0.417 1.000 -ATOM 5743 OH2 TIP3 8687 -18.370 12.397 18.570 -0.834 1.520 -ATOM 5744 H1 TIP3 8687 -18.137 13.066 17.921 0.417 1.000 -ATOM 5745 H2 TIP3 8687 -18.953 12.857 19.192 0.417 1.000 -ATOM 5746 OH2 TIP3 8760 -6.543 0.491 -13.455 -0.834 1.520 -ATOM 5747 H1 TIP3 8760 -6.456 -0.287 -13.041 0.417 1.000 -ATOM 5748 H2 TIP3 8760 -6.801 0.170 -14.399 0.417 1.000 -ATOM 5749 OH2 TIP3 8780 -16.562 -11.901 -19.684 -0.834 1.520 -ATOM 5750 H1 TIP3 8780 -17.453 -12.153 -19.423 0.417 1.000 -ATOM 5751 H2 TIP3 8780 -16.111 -12.225 -18.907 0.417 1.000 -ATOM 5752 OH2 TIP3 8801 -10.984 -4.310 -10.515 -0.834 1.520 -ATOM 5753 H1 TIP3 8801 -10.842 -5.231 -10.299 0.417 1.000 -ATOM 5754 H2 TIP3 8801 -11.463 -4.446 -11.348 0.417 1.000 -ATOM 5755 OH2 TIP3 8813 -15.406 -7.857 16.989 -0.834 1.520 -ATOM 5756 H1 TIP3 8813 -15.792 -7.901 17.928 0.417 1.000 -ATOM 5757 H2 TIP3 8813 -16.238 -7.910 16.469 0.417 1.000 -ATOM 5758 OH2 TIP3 8822 -18.438 -13.464 -7.761 -0.834 1.520 -ATOM 5759 H1 TIP3 8822 -19.040 -12.750 -7.951 0.417 1.000 -ATOM 5760 H2 TIP3 8822 -18.803 -14.173 -8.447 0.417 1.000 -ATOM 5761 OH2 TIP3 8829 -18.686 -10.615 -12.827 -0.834 1.520 -ATOM 5762 H1 TIP3 8829 -18.667 -11.294 -13.556 0.417 1.000 -ATOM 5763 H2 TIP3 8829 -17.944 -10.047 -13.020 0.417 1.000 -ATOM 5764 OH2 TIP3 8833 -17.737 -17.660 10.533 -0.834 1.520 -ATOM 5765 H1 TIP3 8833 -18.006 -18.249 11.233 0.417 1.000 -ATOM 5766 H2 TIP3 8833 -18.591 -17.118 10.429 0.417 1.000 -ATOM 5767 OH2 TIP3 8843 -12.428 -19.580 -16.522 -0.834 1.520 -ATOM 5768 H1 TIP3 8843 -12.364 -19.661 -17.497 0.417 1.000 -ATOM 5769 H2 TIP3 8843 -13.372 -19.766 -16.470 0.417 1.000 -ATOM 5770 OH2 TIP3 8845 -18.867 -12.920 -14.922 -0.834 1.520 -ATOM 5771 H1 TIP3 8845 -18.799 -12.898 -15.863 0.417 1.000 -ATOM 5772 H2 TIP3 8845 -19.642 -13.451 -14.698 0.417 1.000 -ATOM 5773 OH2 TIP3 8848 -9.416 -16.789 7.118 -0.834 1.520 -ATOM 5774 H1 TIP3 8848 -8.820 -16.008 6.914 0.417 1.000 -ATOM 5775 H2 TIP3 8848 -10.016 -16.742 6.356 0.417 1.000 -ATOM 5776 OH2 TIP3 8852 -6.754 -10.545 5.397 -0.834 1.520 -ATOM 5777 H1 TIP3 8852 -6.003 -10.112 5.829 0.417 1.000 -ATOM 5778 H2 TIP3 8852 -6.414 -11.026 4.659 0.417 1.000 -ATOM 5779 OH2 TIP3 8865 -8.485 2.342 -15.173 -0.834 1.520 -ATOM 5780 H1 TIP3 8865 -9.125 1.911 -15.815 0.417 1.000 -ATOM 5781 H2 TIP3 8865 -8.795 1.919 -14.306 0.417 1.000 -ATOM 5782 OH2 TIP3 8869 -14.721 -14.245 -8.543 -0.834 1.520 -ATOM 5783 H1 TIP3 8869 -15.496 -14.667 -9.033 0.417 1.000 -ATOM 5784 H2 TIP3 8869 -14.436 -15.014 -7.981 0.417 1.000 -ATOM 5785 OH2 TIP3 8870 -19.092 -7.587 5.925 -0.834 1.520 -ATOM 5786 H1 TIP3 8870 -19.146 -6.699 5.615 0.417 1.000 -ATOM 5787 H2 TIP3 8870 -19.982 -7.938 5.781 0.417 1.000 -ATOM 5788 OH2 TIP3 8886 9.059 -3.202 -14.555 -0.834 1.520 -ATOM 5789 H1 TIP3 8886 8.687 -2.745 -13.761 0.417 1.000 -ATOM 5790 H2 TIP3 8886 9.347 -2.472 -15.130 0.417 1.000 -ATOM 5791 OH2 TIP3 8887 -11.110 1.041 -3.930 -0.834 1.520 -ATOM 5792 H1 TIP3 8887 -11.302 1.572 -4.700 0.417 1.000 -ATOM 5793 H2 TIP3 8887 -10.200 1.340 -3.756 0.417 1.000 -ATOM 5794 OH2 TIP3 8895 -14.000 -8.717 2.324 -0.834 1.520 -ATOM 5795 H1 TIP3 8895 -14.134 -7.943 2.939 0.417 1.000 -ATOM 5796 H2 TIP3 8895 -14.137 -8.366 1.452 0.417 1.000 -ATOM 5797 OH2 TIP3 8898 -10.631 -6.093 17.857 -0.834 1.520 -ATOM 5798 H1 TIP3 8898 -9.953 -6.488 18.378 0.417 1.000 -ATOM 5799 H2 TIP3 8898 -11.386 -5.956 18.509 0.417 1.000 -ATOM 5800 OH2 TIP3 8903 -3.672 -18.849 -14.215 -0.834 1.520 -ATOM 5801 H1 TIP3 8903 -2.908 -19.461 -14.122 0.417 1.000 -ATOM 5802 H2 TIP3 8903 -4.404 -19.365 -13.851 0.417 1.000 -ATOM 5803 OH2 TIP3 8905 -12.166 -5.592 -12.473 -0.834 1.520 -ATOM 5804 H1 TIP3 8905 -12.895 -5.283 -13.007 0.417 1.000 -ATOM 5805 H2 TIP3 8905 -11.806 -6.286 -13.074 0.417 1.000 -ATOM 5806 OH2 TIP3 8908 -10.655 11.960 -14.096 -0.834 1.520 -ATOM 5807 H1 TIP3 8908 -10.512 11.566 -13.167 0.417 1.000 -ATOM 5808 H2 TIP3 8908 -11.101 12.828 -13.960 0.417 1.000 -ATOM 5809 OH2 TIP3 8918 -11.744 -0.880 13.820 -0.834 1.520 -ATOM 5810 H1 TIP3 8918 -10.977 -1.321 13.483 0.417 1.000 -ATOM 5811 H2 TIP3 8918 -11.615 0.026 13.600 0.417 1.000 -ATOM 5812 OH2 TIP3 8926 -9.781 5.985 -14.833 -0.834 1.520 -ATOM 5813 H1 TIP3 8926 -9.309 5.282 -15.221 0.417 1.000 -ATOM 5814 H2 TIP3 8926 -9.651 5.758 -13.869 0.417 1.000 -ATOM 5815 OH2 TIP3 8927 -15.351 7.814 -11.678 -0.834 1.520 -ATOM 5816 H1 TIP3 8927 -14.731 8.535 -11.993 0.417 1.000 -ATOM 5817 H2 TIP3 8927 -16.228 8.055 -12.040 0.417 1.000 -ATOM 5818 OH2 TIP3 8928 -14.358 -6.559 7.230 -0.834 1.520 -ATOM 5819 H1 TIP3 8928 -14.171 -5.615 7.385 0.417 1.000 -ATOM 5820 H2 TIP3 8928 -13.841 -7.091 7.879 0.417 1.000 -ATOM 5821 OH2 TIP3 8931 -14.333 -8.188 -0.458 -0.834 1.520 -ATOM 5822 H1 TIP3 8931 -14.353 -7.257 -0.503 0.417 1.000 -ATOM 5823 H2 TIP3 8931 -14.111 -8.371 -1.382 0.417 1.000 -ATOM 5824 OH2 TIP3 8932 -7.003 -1.424 -4.793 -0.834 1.520 -ATOM 5825 H1 TIP3 8932 -7.383 -0.807 -5.507 0.417 1.000 -ATOM 5826 H2 TIP3 8932 -6.418 -0.837 -4.263 0.417 1.000 -ATOM 5827 OH2 TIP3 8935 -19.325 -8.426 -9.048 -0.834 1.520 -ATOM 5828 H1 TIP3 8935 -18.403 -8.454 -9.426 0.417 1.000 -ATOM 5829 H2 TIP3 8935 -19.735 -7.850 -9.661 0.417 1.000 -ATOM 5830 OH2 TIP3 8939 -18.020 -3.708 12.037 -0.834 1.520 -ATOM 5831 H1 TIP3 8939 -17.556 -4.251 12.703 0.417 1.000 -ATOM 5832 H2 TIP3 8939 -18.505 -3.002 12.564 0.417 1.000 -ATOM 5833 OH2 TIP3 8940 -14.210 -10.768 13.382 -0.834 1.520 -ATOM 5834 H1 TIP3 8940 -15.013 -11.368 13.560 0.417 1.000 -ATOM 5835 H2 TIP3 8940 -13.414 -11.201 13.717 0.417 1.000 -ATOM 5836 OH2 TIP3 8949 -11.218 -2.813 -17.638 -0.834 1.520 -ATOM 5837 H1 TIP3 8949 -11.996 -3.214 -17.203 0.417 1.000 -ATOM 5838 H2 TIP3 8949 -10.860 -3.578 -18.030 0.417 1.000 -ATOM 5839 OH2 TIP3 8952 -12.407 -10.504 3.616 -0.834 1.520 -ATOM 5840 H1 TIP3 8952 -11.904 -9.827 4.113 0.417 1.000 -ATOM 5841 H2 TIP3 8952 -13.032 -9.951 3.068 0.417 1.000 -ATOM 5842 OH2 TIP3 8954 -13.610 9.666 -12.829 -0.834 1.520 -ATOM 5843 H1 TIP3 8954 -13.211 9.750 -13.709 0.417 1.000 -ATOM 5844 H2 TIP3 8954 -14.087 10.472 -12.758 0.417 1.000 -ATOM 5845 OH2 TIP3 8956 -17.887 -13.484 1.893 -0.834 1.520 -ATOM 5846 H1 TIP3 8956 -17.830 -14.424 2.160 0.417 1.000 -ATOM 5847 H2 TIP3 8956 -17.554 -13.051 2.707 0.417 1.000 -ATOM 5848 OH2 TIP3 8960 -17.255 -8.754 19.163 -0.834 1.520 -ATOM 5849 H1 TIP3 8960 -17.698 -9.363 18.623 0.417 1.000 -ATOM 5850 H2 TIP3 8960 -17.868 -8.584 19.860 0.417 1.000 -ATOM 5851 OH2 TIP3 8961 -8.111 -11.830 15.099 -0.834 1.520 -ATOM 5852 H1 TIP3 8961 -7.310 -11.399 14.804 0.417 1.000 -ATOM 5853 H2 TIP3 8961 -8.769 -11.432 14.464 0.417 1.000 -ATOM 5854 OH2 TIP3 8970 -1.689 3.380 -7.612 -0.834 1.520 -ATOM 5855 H1 TIP3 8970 -1.950 2.472 -7.588 0.417 1.000 -ATOM 5856 H2 TIP3 8970 -1.142 3.546 -6.833 0.417 1.000 -ATOM 5857 OH2 TIP3 8972 -16.200 11.154 -2.205 -0.834 1.520 -ATOM 5858 H1 TIP3 8972 -16.424 11.376 -3.168 0.417 1.000 -ATOM 5859 H2 TIP3 8972 -16.174 10.184 -2.177 0.417 1.000 -ATOM 5860 OH2 TIP3 8973 -16.339 5.314 -14.330 -0.834 1.520 -ATOM 5861 H1 TIP3 8973 -15.828 4.871 -15.004 0.417 1.000 -ATOM 5862 H2 TIP3 8973 -15.952 5.053 -13.531 0.417 1.000 -ATOM 5863 OH2 TIP3 8976 -11.068 7.387 -11.991 -0.834 1.520 -ATOM 5864 H1 TIP3 8976 -11.791 7.019 -11.389 0.417 1.000 -ATOM 5865 H2 TIP3 8976 -10.238 6.895 -11.786 0.417 1.000 -ATOM 5866 OH2 TIP3 8977 -17.417 -3.877 -7.954 -0.834 1.520 -ATOM 5867 H1 TIP3 8977 -16.509 -3.492 -7.941 0.417 1.000 -ATOM 5868 H2 TIP3 8977 -17.688 -3.688 -7.113 0.417 1.000 -ATOM 5869 OH2 TIP3 8988 -17.754 0.883 -15.005 -0.834 1.520 -ATOM 5870 H1 TIP3 8988 -18.389 0.097 -15.115 0.417 1.000 -ATOM 5871 H2 TIP3 8988 -16.921 0.473 -14.805 0.417 1.000 -ATOM 5872 OH2 TIP3 8994 -8.866 13.465 7.648 -0.834 1.520 -ATOM 5873 H1 TIP3 8994 -9.663 13.889 8.046 0.417 1.000 -ATOM 5874 H2 TIP3 8994 -9.280 12.732 7.163 0.417 1.000 -ATOM 5875 OH2 TIP3 9001 -4.087 4.342 15.186 -0.834 1.520 -ATOM 5876 H1 TIP3 9001 -3.693 3.514 15.072 0.417 1.000 -ATOM 5877 H2 TIP3 9001 -5.007 4.201 15.323 0.417 1.000 -ATOM 5878 OH2 TIP3 9013 -13.681 11.624 -9.894 -0.834 1.520 -ATOM 5879 H1 TIP3 9013 -12.913 11.877 -10.459 0.417 1.000 -ATOM 5880 H2 TIP3 9013 -14.236 12.421 -10.039 0.417 1.000 -ATOM 5881 OH2 TIP3 9015 -0.007 10.539 -8.466 -0.834 1.520 -ATOM 5882 H1 TIP3 9015 0.630 11.251 -8.418 0.417 1.000 -ATOM 5883 H2 TIP3 9015 0.051 10.203 -7.536 0.417 1.000 -ATOM 5884 OH2 TIP3 9017 -18.951 -0.123 11.163 -0.834 1.520 -ATOM 5885 H1 TIP3 9017 -19.138 -0.350 12.077 0.417 1.000 -ATOM 5886 H2 TIP3 9017 -19.580 -0.682 10.641 0.417 1.000 -ATOM 5887 OH2 TIP3 9034 -4.246 13.645 -8.855 -0.834 1.520 -ATOM 5888 H1 TIP3 9034 -4.524 13.296 -9.696 0.417 1.000 -ATOM 5889 H2 TIP3 9034 -3.283 13.556 -8.962 0.417 1.000 -ATOM 5890 OH2 TIP3 9035 -17.453 15.847 -8.873 -0.834 1.520 -ATOM 5891 H1 TIP3 9035 -16.878 15.236 -9.378 0.417 1.000 -ATOM 5892 H2 TIP3 9035 -18.266 15.322 -8.781 0.417 1.000 -ATOM 5893 OH2 TIP3 9037 -10.653 15.169 9.389 -0.834 1.520 -ATOM 5894 H1 TIP3 9037 -10.651 14.510 10.081 0.417 1.000 -ATOM 5895 H2 TIP3 9037 -10.866 16.013 9.882 0.417 1.000 -ATOM 5896 OH2 TIP3 9047 -15.284 8.389 -19.042 -0.834 1.520 -ATOM 5897 H1 TIP3 9047 -16.001 8.738 -18.510 0.417 1.000 -ATOM 5898 H2 TIP3 9047 -15.433 7.411 -18.991 0.417 1.000 -ATOM 5899 OH2 TIP3 9054 -12.244 10.127 -6.148 -0.834 1.520 -ATOM 5900 H1 TIP3 9054 -11.679 10.781 -6.711 0.417 1.000 -ATOM 5901 H2 TIP3 9054 -13.136 10.405 -6.437 0.417 1.000 -ATOM 5902 OH2 TIP3 9055 -16.995 9.797 -16.973 -0.834 1.520 -ATOM 5903 H1 TIP3 9055 -17.170 8.988 -16.504 0.417 1.000 -ATOM 5904 H2 TIP3 9055 -16.221 10.190 -16.587 0.417 1.000 -ATOM 5905 OH2 TIP3 9056 -10.546 17.563 5.092 -0.834 1.520 -ATOM 5906 H1 TIP3 9056 -10.652 16.598 4.989 0.417 1.000 -ATOM 5907 H2 TIP3 9056 -10.017 17.607 5.931 0.417 1.000 -ATOM 5908 OH2 TIP3 9057 -7.410 0.617 8.532 -0.834 1.520 -ATOM 5909 H1 TIP3 9057 -7.622 0.494 9.511 0.417 1.000 -ATOM 5910 H2 TIP3 9057 -8.213 0.273 8.098 0.417 1.000 -ATOM 5911 OH2 TIP3 9082 -11.844 -2.436 -8.480 -0.834 1.520 -ATOM 5912 H1 TIP3 9082 -11.645 -1.655 -9.079 0.417 1.000 -ATOM 5913 H2 TIP3 9082 -11.666 -3.180 -9.126 0.417 1.000 -ATOM 5914 OH2 TIP3 9084 -11.252 9.168 14.621 -0.834 1.520 -ATOM 5915 H1 TIP3 9084 -11.545 9.630 13.875 0.417 1.000 -ATOM 5916 H2 TIP3 9084 -12.003 8.592 14.797 0.417 1.000 -ATOM 5917 OH2 TIP3 9093 -10.008 15.829 -16.703 -0.834 1.520 -ATOM 5918 H1 TIP3 9093 -10.013 16.093 -17.662 0.417 1.000 -ATOM 5919 H2 TIP3 9093 -9.070 15.998 -16.520 0.417 1.000 -ATOM 5920 OH2 TIP3 9096 -17.085 -5.803 -16.607 -0.834 1.520 -ATOM 5921 H1 TIP3 9096 -17.845 -5.586 -16.084 0.417 1.000 -ATOM 5922 H2 TIP3 9096 -17.281 -6.667 -16.989 0.417 1.000 -ATOM 5923 OH2 TIP3 9127 -5.246 18.382 8.553 -0.834 1.520 -ATOM 5924 H1 TIP3 9127 -5.561 19.111 7.984 0.417 1.000 -ATOM 5925 H2 TIP3 9127 -5.914 18.448 9.262 0.417 1.000 -ATOM 5926 OH2 TIP3 9165 -12.613 -19.814 6.924 -0.834 1.520 -ATOM 5927 H1 TIP3 9165 -12.998 -19.069 6.433 0.417 1.000 -ATOM 5928 H2 TIP3 9165 -13.054 -19.750 7.843 0.417 1.000 -ATOM 5929 OH2 TIP3 9170 7.884 19.129 1.844 -0.834 1.520 -ATOM 5930 H1 TIP3 9170 8.466 18.885 1.122 0.417 1.000 -ATOM 5931 H2 TIP3 9170 7.278 18.412 1.870 0.417 1.000 -ATOM 5932 OH2 TIP3 9212 -15.144 -18.317 17.499 -0.834 1.520 -ATOM 5933 H1 TIP3 9212 -14.489 -18.228 16.838 0.417 1.000 -ATOM 5934 H2 TIP3 9212 -14.650 -18.818 18.202 0.417 1.000 -ATOM 5935 OH2 TIP3 9226 -11.422 -19.168 -13.649 -0.834 1.520 -ATOM 5936 H1 TIP3 9226 -12.072 -19.470 -14.307 0.417 1.000 -ATOM 5937 H2 TIP3 9226 -11.437 -18.199 -13.665 0.417 1.000 -ATOM 5938 OH2 TIP3 9241 -12.713 -9.828 -6.185 -0.834 1.520 -ATOM 5939 H1 TIP3 9241 -13.369 -10.527 -5.923 0.417 1.000 -ATOM 5940 H2 TIP3 9241 -11.881 -10.331 -6.148 0.417 1.000 -ATOM 5941 OH2 TIP3 9250 -16.653 -9.513 -0.186 -0.834 1.520 -ATOM 5942 H1 TIP3 9250 -17.208 -9.045 -0.812 0.417 1.000 -ATOM 5943 H2 TIP3 9250 -15.736 -9.361 -0.544 0.417 1.000 -END diff --git a/tests/opencl/lbm/main.cc b/tests/opencl/lbm/main.cc index 1d825239..58a930e9 100644 --- a/tests/opencl/lbm/main.cc +++ b/tests/opencl/lbm/main.cc @@ -173,14 +173,10 @@ void MAIN_initialize(const MAIN_Param *param, const OpenCL_Param *prm) { pb_SwitchToTimer(&timers, pb_TimerID_COPY); - printf("OK+\n"); - // Setup DEVICE datastructures OpenCL_LBM_allocateGrid(prm, &OpenCL_srcGrid); OpenCL_LBM_allocateGrid(prm, &OpenCL_dstGrid); - printf("OK-\n"); - // Initialize DEVICE datastructures OpenCL_LBM_initializeGrid(prm, OpenCL_srcGrid, TEMP_srcGrid); OpenCL_LBM_initializeGrid(prm, OpenCL_dstGrid, TEMP_dstGrid); diff --git a/tests/opencl/matmul/kernel.cl b/tests/opencl/matmul/kernel.cl deleted file mode 100644 index ea9b2156..00000000 --- a/tests/opencl/matmul/kernel.cl +++ /dev/null @@ -1,73 +0,0 @@ -__kernel void matmul(__global float *A, - __global float *B, - __global float *C, - const unsigned int N, - __local float *localA, - __local float *localB) -{ - int row = get_global_id(1); - int col = get_global_id(0); - int localRow = get_local_id(1); - int localCol = get_local_id(0); - int localSize = get_local_size(0); // assuming square local size - - float sum = 0.0f; - - // Loop over all blocks of both matrices - for (int k = 0; k < N; k += localSize) { - // Load block of matrix A to local memory - localA[localRow * localSize + localCol] = A[row * N + k + localCol]; - - // Load block of matrix B to local memory, adjusting for column-major access - localB[localRow * localSize + localCol] = B[(k + localRow) * N + col]; - - // Synchronize to make sure the tiles are loaded - barrier(CLK_LOCAL_MEM_FENCE); - - // Multiply the two matrix blocks and accumulate result - for (int j = 0; j < localSize; j++) { - sum += localA[localRow * localSize + j] * localB[j * localSize + localCol]; - } - - // Synchronize before loading the next block - barrier(CLK_LOCAL_MEM_FENCE); - } - - C[row * N + col] = sum; -} - -/*__kernel void matmul(__global float *A, __global float *B, __global float *C, const unsigned int N) -{ - int globalRow = get_global_id(1); - int globalCol = get_global_id(0); - int localRow = get_local_id(1); - int localCol = get_local_id(0); - - // Static local memory declaration - __local float localA[16][16]; - __local float localB[16][16]; - - float sum = 0.0f; - - // Iterate over blocks - for (int k = 0; k < N; k += 16) { - // Load a block of matrix A into local memory - localA[localRow][localCol] = A[globalRow * N + k + localCol]; - - // Load a block of matrix B into local memory - localB[localRow][localCol] = B[(k + localRow) * N + globalCol]; - - // Ensure the entire block is loaded - barrier(CLK_LOCAL_MEM_FENCE); - - // Compute multiplication for this block - for (int j = 0; j < 16; j++) { - sum += localA[localRow][j] * localB[j][localCol]; - } - - // Wait until all threads have computed before loading the next block - barrier(CLK_LOCAL_MEM_FENCE); - } - - C[globalRow * N + globalCol] = sum; -}*/ \ No newline at end of file diff --git a/tests/opencl/mri-q/32_32_32_dataset.bin b/tests/opencl/mri-q/32_32_32_dataset.bin deleted file mode 100755 index db8385bb..00000000 Binary files a/tests/opencl/mri-q/32_32_32_dataset.bin and /dev/null differ diff --git a/tests/opencl/mri-q/Makefile b/tests/opencl/mri-q/Makefile deleted file mode 100644 index dbff34f9..00000000 --- a/tests/opencl/mri-q/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -PROJECT = mri-q - -SRCS = main.cc args.c parboil_opencl.c ocl.c gpu_info.c file.cc computeQ.c - -CXXFLAGS += -I. - -OPTS ?= - -include ../common.mk diff --git a/tests/opencl/mri-q/args.c b/tests/opencl/mri-q/args.c deleted file mode 100644 index 9d751e29..00000000 --- a/tests/opencl/mri-q/args.c +++ /dev/null @@ -1,617 +0,0 @@ - -#include -#include -#include -#include -#include -#include - -/*****************************************************************************/ -/* Memory management routines */ - -/* Free an array of owned strings. */ -void -pb_FreeStringArray(char **string_array) -{ - char **p; - - if (!string_array) return; - for (p = string_array; *p; p++) free(*p); - free(string_array); -} - -struct pb_PlatformParam * -pb_PlatformParam(char *name, char *version) -{ - if (name == NULL) { - fprintf(stderr, "pb_PlatformParam: Invalid argument\n"); - exit(-1); - } - - struct pb_PlatformParam *ret = - (struct pb_PlatformParam *)malloc(sizeof (struct pb_PlatformParam)); - - ret->name = name; - ret->version = version; - return ret; -} - -void -pb_FreePlatformParam(struct pb_PlatformParam *p) -{ - if (p == NULL) return; - - free(p->name); - free(p->version); - free(p); -} - -struct pb_DeviceParam * -pb_DeviceParam_index(int index) -{ - struct pb_DeviceParam *ret = - (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); - ret->criterion = pb_Device_INDEX; - ret->index = index; - return ret; -} - -struct pb_DeviceParam * -pb_DeviceParam_cpu(void) -{ - struct pb_DeviceParam *ret = - (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); - ret->criterion = pb_Device_CPU; - return ret; -} - -struct pb_DeviceParam * -pb_DeviceParam_gpu(void) -{ - struct pb_DeviceParam *ret = - (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); - ret->criterion = pb_Device_GPU; - return ret; -} - -struct pb_DeviceParam * -pb_DeviceParam_accelerator(void) -{ - struct pb_DeviceParam *ret = - (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); - ret->criterion = pb_Device_ACCELERATOR; - return ret; -} - -struct pb_DeviceParam * -pb_DeviceParam_name(char *name) -{ - struct pb_DeviceParam *ret = - (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); - ret->criterion = pb_Device_NAME; - ret->name = name; - return ret; -} - -void -pb_FreeDeviceParam(struct pb_DeviceParam *p) -{ - if (p == NULL) return; - - switch(p->criterion) { - case pb_Device_NAME: - free(p->name); - break; - case pb_Device_INDEX: - case pb_Device_CPU: - case pb_Device_ACCELERATOR: - break; - default: - fprintf(stderr, "pb_FreeDeviceParam: Invalid argument\n"); - exit(-1); - } -} - -void -pb_FreeParameters(struct pb_Parameters *p) -{ - free(p->outFile); - pb_FreeStringArray(p->inpFiles); - pb_FreePlatformParam(p->platform); - pb_FreeDeviceParam(p->device); - free(p); -} - -/*****************************************************************************/ - -/* Parse a comma-delimited list of strings into an - * array of strings. */ -static char ** -read_string_array(char *in) -{ - char **ret; - int i; - int count; /* Number of items in the input */ - char *substring; /* Current substring within 'in' */ - - /* Count the number of items in the string */ - count = 1; - for (i = 0; in[i]; i++) if (in[i] == ',') count++; - - /* Allocate storage */ - ret = (char **)malloc((count + 1) * sizeof(char *)); - - /* Create copies of the strings from the list */ - substring = in; - for (i = 0; i < count; i++) { - char *substring_end; - int substring_length; - - /* Find length of substring */ - for (substring_end = substring; - (*substring_end != ',') && (*substring_end != 0); - substring_end++); - - substring_length = substring_end - substring; - - /* Allocate memory and copy the substring */ - ret[i] = (char *)malloc(substring_length + 1); - memcpy(ret[i], substring, substring_length); - ret[i][substring_length] = 0; - - /* go to next substring */ - substring = substring_end + 1; - } - ret[i] = NULL; /* Write the sentinel value */ - - return ret; -} - -static void -report_parse_error(const char *str) -{ - fputs(str, stderr); -} - -/* Interpret a string as a 'pb_DeviceParam' value. - * Return a pointer to a new value, or NULL on failure. - */ -static struct pb_DeviceParam * -read_device_param(char *str) -{ - /* Try different ways of interpreting 'device_string' until one works */ - - /* If argument is an integer, then interpret it as a device index */ - errno = 0; - char *end; - long device_int = strtol(str, &end, 10); - if (!errno) { - /* Negative numbers are not valid */ - if (device_int < 0 || device_int > INT_MAX) return NULL; - - return pb_DeviceParam_index(device_int); - } - - /* Match against predefined strings */ - if (strcmp(str, "CPU") == 0) - return pb_DeviceParam_cpu(); - if (strcmp(str, "GPU") == 0) - return pb_DeviceParam_gpu(); - if (strcmp(str, "ACCELERATOR") == 0) - return pb_DeviceParam_accelerator(); - - /* Assume any other string is a device name */ - return pb_DeviceParam_name(strdup(str)); -} - -/* Interpret a string as a 'pb_PlatformParam' value. - * Return a pointer to a new value, or NULL on failure. - */ -static struct pb_PlatformParam * -read_platform_param(char *str) -{ - int separator_index; /* Index of the '-' character separating - * name and version number. It's -1 if - * there's no '-' character. */ - - /* Find the last occurrence of '-' in 'str' */ - { - char *cur; - separator_index = -1; - for (cur = str; *cur; cur++) { - if (*cur == '-') separator_index = cur - str; - } - } - - /* The platform name is either the entire string, or all characters before - * the separator */ - int name_length = separator_index == -1 ? strlen(str) : separator_index; - char *name_str = (char *)malloc(name_length + 1); - memcpy(name_str, str, name_length); - name_str[name_length] = 0; - - /* The version is either NULL, or all characters after the separator */ - char *version_str; - if (separator_index == -1) { - version_str = NULL; - } - else { - const char *version_input_str = str + separator_index + 1; - int version_length = strlen(version_input_str); - - version_str = (char *)malloc(version_length + 1); - memcpy(version_str, version_input_str, version_length); - version_str[version_length] = 0; - } - - /* Create output structure */ - return pb_PlatformParam(name_str, version_str); -} - -/****************************************************************************/ -/* Argument parsing state */ - -/* Argument parsing state. - * - * Arguments that are interpreted by the argument parser are removed from - * the list. Variables 'argc' and 'argn' do not count arguments that have - * been removed. - * - * During argument parsing, the array of arguments is compacted, overwriting - * the erased arguments. Variable 'argv_put' points to the array element - * where the next argument will be written. Variable 'argv_get' points to - * the array element where the next argument will be read from. - */ -struct argparse { - int argc; /* Number of arguments. Mutable. */ - int argn; /* Current argument index. */ - char **argv_get; /* Argument value being read. */ - char **argv_put; /* Argument value being written. - * argv_put <= argv_get. */ -}; - -static void -initialize_argparse(struct argparse *ap, int argc, char **argv) -{ - ap->argc = argc; - ap->argn = 0; - ap->argv_get = ap->argv_put = argv; -} - -/* Finish argument parsing, without processing the remaining arguments. - * Write new argument count into _argc. */ -static void -finalize_argparse(struct argparse *ap, int *_argc, char **argv) -{ - /* Move the remaining arguments */ - for(; ap->argn < ap->argc; ap->argn++) - *ap->argv_put++ = *ap->argv_get++; - - /* Update the argument count */ - *_argc = ap->argc; - - /* Insert a terminating NULL */ - argv[ap->argc] = NULL; -} - -/* Delete the current argument. The argument will not be visible - * when argument parsing is done. */ -static void -delete_argument(struct argparse *ap) -{ - if (ap->argn >= ap->argc) { - fprintf(stderr, "delete_argument\n"); - } - ap->argc--; - ap->argv_get++; -} - -/* Go to the next argument. Also, move the current argument to its - * final location in argv. */ -static void -next_argument(struct argparse *ap) -{ - if (ap->argn >= ap->argc) { - fprintf(stderr, "next_argument\n"); - } - /* Move argument to its new location. */ - *ap->argv_put++ = *ap->argv_get++; - ap->argn++; -} - -static int -is_end_of_arguments(struct argparse *ap) -{ - return ap->argn == ap->argc; -} - -/* Get the current argument */ -static char * -get_argument(struct argparse *ap) -{ - return *ap->argv_get; -} - -/* Get the current argument, and also delete it */ -static char * -consume_argument(struct argparse *ap) -{ - char *ret = get_argument(ap); - delete_argument(ap); - return ret; -} - -/****************************************************************************/ - -/* The result of parsing a command-line argument */ -typedef enum { - ARGPARSE_OK, /* Success */ - ARGPARSE_ERROR, /* Error */ - ARGPARSE_DONE /* Success, and do not continue parsing */ -} result; - -typedef result parse_action(struct argparse *ap, struct pb_Parameters *params); - - -/* A command-line option */ -struct option { - char short_name; /* If not 0, the one-character - * name of this option */ - const char *long_name; /* If not NULL, the long name of this option */ - parse_action *action; /* What to do when this option occurs. - * Sentinel value is NULL. - */ -}; - -/* Output file - * - * -o FILE - */ -static result -parse_output_file(struct argparse *ap, struct pb_Parameters *params) -{ - if (is_end_of_arguments(ap)) - { - report_parse_error("Expecting file name after '-o'\n"); - return ARGPARSE_ERROR; - } - - /* Replace the output file name */ - free(params->outFile); - params->outFile = strdup(consume_argument(ap)); - - return ARGPARSE_OK; -} - -/* Input files - * - * -i FILE,FILE,... - */ -static result -parse_input_files(struct argparse *ap, struct pb_Parameters *params) -{ - if (is_end_of_arguments(ap)) - { - report_parse_error("Expecting file name after '-i'\n"); - return ARGPARSE_ERROR; - } - - /* Replace the input file list */ - pb_FreeStringArray(params->inpFiles); - params->inpFiles = read_string_array(consume_argument(ap)); - return ARGPARSE_OK; -} - -/* End of options - * - * -- - */ - -static result -parse_end_options(struct argparse *ap, struct pb_Parameters *params) -{ - return ARGPARSE_DONE; -} - -/* OpenCL device - * - * --device X - */ - -static result -parse_device(struct argparse *ap, struct pb_Parameters *params) -{ - /* Read the next argument, which specifies a device */ - - if (is_end_of_arguments(ap)) - { - report_parse_error("Expecting device specification after '--device'\n"); - return ARGPARSE_ERROR; - } - - char *device_string = consume_argument(ap); - struct pb_DeviceParam *device_param = read_device_param(device_string); - - if (!device_param) { - report_parse_error("Unrecognized device specification format on command line\n"); - return ARGPARSE_ERROR; - } - - /* Save the result */ - pb_FreeDeviceParam(params->device); - params->device = device_param; - - return ARGPARSE_OK; -} - -static result -parse_platform(struct argparse *ap, struct pb_Parameters *params) -{ - /* Read the next argument, which specifies a platform */ - - if (is_end_of_arguments(ap)) - { - report_parse_error("Expecting device specification after '--platform'\n"); - return ARGPARSE_ERROR; - } - - char *platform_string = consume_argument(ap); - struct pb_PlatformParam *platform_param = read_platform_param(platform_string); - - if (!platform_param) { - report_parse_error("Unrecognized platform specification format on command line\n"); - return ARGPARSE_ERROR; - } - - /* Save the result */ - pb_FreePlatformParam(params->platform); - params->platform = platform_param; - - return ARGPARSE_OK; -} - - -static struct option options[] = { - { 'o', NULL, &parse_output_file }, - { 'i', NULL, &parse_input_files }, - { '-', NULL, &parse_end_options }, - { 0, "device", &parse_device }, - { 0, "platform", &parse_platform }, - { 0, NULL, NULL } -}; - -static int -is_last_option(struct option *op) -{ - return op->action == NULL; -} - -/****************************************************************************/ - -/* Parse command-line parameters. - * Return zero on error, nonzero otherwise. - * On error, the other outputs may be invalid. - * - * The information collected from parameters is used to update - * 'ret'. 'ret' should be initialized. - * - * '_argc' and 'argv' are updated to contain only the unprocessed arguments. - */ -static int -pb_ParseParameters (struct pb_Parameters *ret, int *_argc, char **argv) -{ - char *err_message; - struct argparse ap; - - /* Each argument */ - initialize_argparse(&ap, *_argc, argv); - while(!is_end_of_arguments(&ap)) { - result arg_result; /* Result of parsing this option */ - char *arg = get_argument(&ap); - - /* Process this argument */ - if (arg[0] == '-') { - /* Single-character flag */ - if ((arg[1] != 0) && (arg[2] == 0)) { - delete_argument(&ap); /* This argument is consumed here */ - - /* Find a matching short option */ - struct option *op; - for (op = options; !is_last_option(op); op++) { - if (op->short_name == arg[1]) { - arg_result = (*op->action)(&ap, ret); - goto option_was_processed; - } - } - - /* No option matches */ - report_parse_error("Unexpected command-line parameter\n"); - arg_result = ARGPARSE_ERROR; - goto option_was_processed; - } - - /* Long flag */ - if (arg[1] == '-') { - delete_argument(&ap); /* This argument is consumed here */ - - /* Find a matching long option */ - struct option *op; - for (op = options; !is_last_option(op); op++) { - if (op->long_name && strcmp(&arg[2], op->long_name) == 0) { - arg_result = (*op->action)(&ap, ret); - goto option_was_processed; - } - } - - /* No option matches */ - report_parse_error("Unexpected command-line parameter\n"); - arg_result = ARGPARSE_ERROR; - goto option_was_processed; - } - } - else { - /* Other arguments are ignored */ - next_argument(&ap); - arg_result = ARGPARSE_OK; - goto option_was_processed; - } - - option_was_processed: - /* Decide what to do next based on 'arg_result' */ - switch(arg_result) { - case ARGPARSE_OK: - /* Continue processing */ - break; - - case ARGPARSE_ERROR: - /* Error exit from the function */ - return 0; - - case ARGPARSE_DONE: - /* Normal exit from the argument parsing loop */ - goto end_of_options; - } - } /* end for each argument */ - - /* If all arguments were processed, then normal exit from the loop */ - - end_of_options: - finalize_argparse(&ap, _argc, argv); - return 1; -} - -/*****************************************************************************/ -/* Other exported functions */ - -struct pb_Parameters * -pb_ReadParameters(int *_argc, char **argv) -{ - struct pb_Parameters *ret = - (struct pb_Parameters *)malloc(sizeof(struct pb_Parameters)); - - /* Initialize the parameters structure */ - ret->outFile = NULL; - ret->inpFiles = (char **)malloc(sizeof(char *)); - ret->inpFiles[0] = NULL; - ret->platform = NULL; - ret->device = NULL; - - /* Read parameters and update _argc, argv */ - if (!pb_ParseParameters(ret, _argc, argv)) { - /* Parse error */ - pb_FreeParameters(ret); - return NULL; - } - - return ret; -} - -int -pb_Parameters_CountInputs(struct pb_Parameters *p) -{ - int n; - - for (n = 0; p->inpFiles[n]; n++); - return n; -} - diff --git a/tests/opencl/mri-q/computeQ.c b/tests/opencl/mri-q/computeQ.c deleted file mode 100644 index 65ed6f4d..00000000 --- a/tests/opencl/mri-q/computeQ.c +++ /dev/null @@ -1,118 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2007 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#include -#include -#include -#include "ocl.h" -#include "macros.h" -#include "computeQ.h" -#include "parboil.h" - -#define NC 1 - -void computePhiMag_GPU(int numK,cl_mem phiR_d,cl_mem phiI_d,cl_mem phiMag_d,clPrmtr* clPrm) -{ - int phiMagBlocks = numK / KERNEL_PHI_MAG_THREADS_PER_BLOCK; - if (numK % KERNEL_PHI_MAG_THREADS_PER_BLOCK) - phiMagBlocks++; - - size_t DimPhiMagBlock = KERNEL_PHI_MAG_THREADS_PER_BLOCK; - size_t DimPhiMagGrid = phiMagBlocks*KERNEL_PHI_MAG_THREADS_PER_BLOCK; - - cl_int clStatus; - clStatus = clSetKernelArg(clPrm->clKernel,0,sizeof(cl_mem),&phiR_d); - clStatus = clSetKernelArg(clPrm->clKernel,1,sizeof(cl_mem),&phiI_d); - clStatus = clSetKernelArg(clPrm->clKernel,2,sizeof(cl_mem),&phiMag_d); - clStatus = clSetKernelArg(clPrm->clKernel,3,sizeof(int),&numK); - CHECK_ERROR("clSetKernelArg") - - clStatus = clEnqueueNDRangeKernel(clPrm->clCommandQueue,clPrm->clKernel,1,NULL,&DimPhiMagGrid,&DimPhiMagBlock,0,NULL,NULL); - CHECK_ERROR("clEnqueueNDRangeKernel") -} - -static -unsigned long long int -readElapsedTime(cl_event internal) -{ - cl_int status; - cl_ulong t_begin, t_end; - status = clGetEventProfilingInfo(internal, CL_PROFILING_COMMAND_START, - sizeof(cl_ulong), &t_begin, NULL); - if (status != CL_SUCCESS) return 0; - status = clGetEventProfilingInfo(internal, CL_PROFILING_COMMAND_END, - sizeof(cl_ulong), &t_end, NULL); - if (status != CL_SUCCESS) return 0; - return (unsigned long long int)(t_end - t_begin); -} - - -void computeQ_GPU (int numK,int numX, - cl_mem x_d, cl_mem y_d, cl_mem z_d, - struct kValues* kVals, - cl_mem Qr_d, cl_mem Qi_d, - clPrmtr* clPrm) -{ - int QGrids = numK / KERNEL_Q_K_ELEMS_PER_GRID; - if (numK % KERNEL_Q_K_ELEMS_PER_GRID) - QGrids++; - int QBlocks = numX / KERNEL_Q_THREADS_PER_BLOCK; - if (numX % KERNEL_Q_THREADS_PER_BLOCK) - QBlocks++; - - size_t DimQBlock = KERNEL_Q_THREADS_PER_BLOCK/NC; - size_t DimQGrid = QBlocks*KERNEL_Q_THREADS_PER_BLOCK/NC; - - cl_int clStatus; - cl_mem ck; - ck = clCreateBuffer(clPrm->clContext,CL_MEM_READ_WRITE,KERNEL_Q_K_ELEMS_PER_GRID*sizeof(struct kValues),NULL,&clStatus); - - int QGrid; - for (QGrid = 0; QGrid < QGrids; QGrid++) { - // Put the tile of K values into constant mem - int QGridBase = QGrid * KERNEL_Q_K_ELEMS_PER_GRID; - struct kValues* kValsTile = kVals + QGridBase; - int numElems = MIN(KERNEL_Q_K_ELEMS_PER_GRID, numK - QGridBase); - - clStatus = clEnqueueWriteBuffer(clPrm->clCommandQueue,ck,CL_TRUE,0,numElems*sizeof(struct kValues),kValsTile,0,NULL,NULL); - CHECK_ERROR("clEnqueueWriteBuffer") - - clStatus = clSetKernelArg(clPrm->clKernel,0,sizeof(int),&numK); - clStatus = clSetKernelArg(clPrm->clKernel,1,sizeof(int),&QGridBase); - clStatus = clSetKernelArg(clPrm->clKernel,2,sizeof(cl_mem),&x_d); - clStatus = clSetKernelArg(clPrm->clKernel,3,sizeof(cl_mem),&y_d); - clStatus = clSetKernelArg(clPrm->clKernel,4,sizeof(cl_mem),&z_d); - clStatus = clSetKernelArg(clPrm->clKernel,5,sizeof(cl_mem),&Qr_d); - clStatus = clSetKernelArg(clPrm->clKernel,6,sizeof(cl_mem),&Qi_d); - clStatus = clSetKernelArg(clPrm->clKernel,7,sizeof(cl_mem),&ck); - CHECK_ERROR("clSetKernelArg") - - printf ("Grid: %d, Block: %d\n", DimQGrid, DimQBlock); - - #define TIMED_EXECUTION - #ifdef TIMED_EXECUTION - cl_event e; - clStatus = clEnqueueNDRangeKernel(clPrm->clCommandQueue,clPrm->clKernel,1,NULL,&DimQGrid,&DimQBlock,0,NULL,&e); - CHECK_ERROR("clEnqueueNDRangeKernel") - clWaitForEvents(1, &e); - printf ("%llu\n", readElapsedTime(e)); - #else - clStatus = clEnqueueNDRangeKernel(clPrm->clCommandQueue,clPrm->clKernel,1,NULL,&DimQGrid,&DimQBlock,0,NULL,NULL); - CHECK_ERROR("clEnqueueNDRangeKernel") - #endif - } -} - -void createDataStructsCPU(int numK, int numX, float** phiMag, - float** Qr, float** Qi) -{ - *phiMag = (float* ) memalign(16, numK * sizeof(float)); - *Qr = (float*) memalign(16, numX * sizeof (float)); - *Qi = (float*) memalign(16, numX * sizeof (float)); -} - diff --git a/tests/opencl/mri-q/computeQ.h b/tests/opencl/mri-q/computeQ.h deleted file mode 100644 index 3312902f..00000000 --- a/tests/opencl/mri-q/computeQ.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef __COMPUTEQ__ -#define __COMPUTEQ__ - -#ifdef __cplusplus -extern "C" { -#endif - -void computePhiMag_GPU(int numK,cl_mem phiR_d,cl_mem phiI_d,cl_mem phiMag_d,clPrmtr* clPrm); -void computeQ_GPU (int numK,int numX, - cl_mem x_d, cl_mem y_d, cl_mem z_d, - struct kValues* kVals, - cl_mem Qr_d, cl_mem Qi_d, - clPrmtr* clPrm); - -void createDataStructsCPU(int numK, int numX, float** phiMag, - float** Qr, float** Qi); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/tests/opencl/mri-q/file.cc b/tests/opencl/mri-q/file.cc deleted file mode 100644 index 15b07075..00000000 --- a/tests/opencl/mri-q/file.cc +++ /dev/null @@ -1,78 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2007 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -//#include -#include -#include -#include -#include - -#include "file.h" - -#if __BYTE_ORDER != __LITTLE_ENDIAN -# error "File I/O is not implemented for this system: wrong endianness." -#endif - -extern "C" -void inputData(char* fName, int* _numK, int* _numX, - float** kx, float** ky, float** kz, - float** x, float** y, float** z, - float** phiR, float** phiI) -{ - int numK, numX; - FILE* fid = fopen(fName, "r"); - - if (fid == NULL) - { - fprintf(stderr, "Cannot open input file\n"); - exit(-1); - } - fread (&numK, sizeof (int), 1, fid); - *_numK = numK; - fread (&numX, sizeof (int), 1, fid); - *_numX = numX; - *kx = (float *) memalign(16, numK * sizeof (float)); - fread (*kx, sizeof (float), numK, fid); - *ky = (float *) memalign(16, numK * sizeof (float)); - fread (*ky, sizeof (float), numK, fid); - *kz = (float *) memalign(16, numK * sizeof (float)); - fread (*kz, sizeof (float), numK, fid); - *x = (float *) memalign(16, numX * sizeof (float)); - fread (*x, sizeof (float), numX, fid); - *y = (float *) memalign(16, numX * sizeof (float)); - fread (*y, sizeof (float), numX, fid); - *z = (float *) memalign(16, numX * sizeof (float)); - fread (*z, sizeof (float), numX, fid); - *phiR = (float *) memalign(16, numK * sizeof (float)); - fread (*phiR, sizeof (float), numK, fid); - *phiI = (float *) memalign(16, numK * sizeof (float)); - fread (*phiI, sizeof (float), numK, fid); - fclose (fid); -} - -extern "C" -void outputData(char* fName, float* outR, float* outI, int numX) -{ - FILE* fid = fopen(fName, "w"); - uint32_t tmp32; - - if (fid == NULL) - { - fprintf(stderr, "Cannot open output file\n"); - exit(-1); - } - - /* Write the data size */ - tmp32 = numX; - fwrite(&tmp32, sizeof(uint32_t), 1, fid); - - /* Write the reconstructed data */ - fwrite (outR, sizeof (float), numX, fid); - fwrite (outI, sizeof (float), numX, fid); - fclose (fid); -} diff --git a/tests/opencl/mri-q/file.h b/tests/opencl/mri-q/file.h deleted file mode 100644 index c6a61ef4..00000000 --- a/tests/opencl/mri-q/file.h +++ /dev/null @@ -1,22 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2007 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#ifdef __cplusplus -extern "C" { -#endif - -void inputData(char* fName, int* _numK, int* _numX, - float** kx, float** ky, float** kz, - float** x, float** y, float** z, - float** phiR, float** phiI); - -void outputData(char* fName, float* outR, float* outI, int numX); - -#ifdef __cplusplus -} -#endif diff --git a/tests/opencl/mri-q/gpu_info.c b/tests/opencl/mri-q/gpu_info.c deleted file mode 100644 index 4d641f81..00000000 --- a/tests/opencl/mri-q/gpu_info.c +++ /dev/null @@ -1,55 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ -//#include -#include -#include -#include -#include - -#include "gpu_info.h" - -void compute_active_thread(size_t *thread, - size_t *grid, - int task, - int pad, - int major, - int minor, - int sm) -{ - int max_thread; - int max_block=8; - if(major==1) - { - if(minor>=2) - max_thread=1024; - else - max_thread=768; - } - else if(major==2) - max_thread=1536; - else - //newer GPU //keep using 2.0 - max_thread=1536; - - int _grid; - int _thread; - - if(task*pad>sm*max_thread) - { - _thread=max_thread/max_block; - _grid = ((task*pad+_thread-1)/_thread)*_thread; - } - else - { - _thread=pad; - _grid=task*pad; - } - - thread[0]=_thread; - grid[0]=_grid; -} diff --git a/tests/opencl/mri-q/gpu_info.h b/tests/opencl/mri-q/gpu_info.h deleted file mode 100644 index 5d433968..00000000 --- a/tests/opencl/mri-q/gpu_info.h +++ /dev/null @@ -1,28 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2010 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -#ifndef __GPUINFOH__ -#define __GPUINFOH__ - -#ifdef __cplusplus -extern "C" { -#endif - -void compute_active_thread(size_t *thread, - size_t *grid, - int task, - int pad, - int major, - int minor, - int sm); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/tests/opencl/mri-q/kernel.cl b/tests/opencl/mri-q/kernel.cl deleted file mode 100644 index 39a1842e..00000000 --- a/tests/opencl/mri-q/kernel.cl +++ /dev/null @@ -1,51 +0,0 @@ -#include "macros.h" - -__kernel void -ComputePhiMag_GPU(__global float* phiR, __global float* phiI, __global float* phiMag, int numK) { - int indexK = get_global_id(0); - float real = indexK; - float imag = indexK; - if (indexK < numK) { - /*float*/ real = phiR[indexK]; - /*float*/ imag = phiI[indexK]; - phiMag[indexK] = real*real + imag*imag; - } -} - -__kernel void -ComputeQ_GPU(int numK, int kGlobalIndex, - __global float* x, __global float* y, __global float* z, - __global float* Qr, __global float* Qi, __global struct kValues* ck) -{ - float sX; - float sY; - float sZ; - float sQr; - float sQi; - - // Determine the element of the X arrays computed by this thread - int xIndex = get_group_id(0)*KERNEL_Q_THREADS_PER_BLOCK + get_local_id(0); - - // Read block's X values from global mem to shared mem - sX = x[xIndex]; - sY = y[xIndex]; - sZ = z[xIndex]; - sQr = Qr[xIndex]; - sQi = Qi[xIndex]; - - int kIndex = 0; - for (; (kIndex < KERNEL_Q_K_ELEMS_PER_GRID); kIndex++) { - if (kGlobalIndex < numK) { - float expArg; - expArg = PIx2 * (ck[kIndex].Kx * sX + - ck[kIndex].Ky * sY + - ck[kIndex].Kz * sZ); - sQr = sQr + ck[kIndex].PhiMag * cos(expArg); // native_cos(expArg); - sQi = sQi + ck[kIndex].PhiMag * sin(expArg); // native_sin(expArg); - } - kGlobalIndex++; - } - - Qr[xIndex] = sQr; - Qi[xIndex] = sQi; -} diff --git a/tests/opencl/mri-q/macros.h b/tests/opencl/mri-q/macros.h deleted file mode 100644 index 501ead7e..00000000 --- a/tests/opencl/mri-q/macros.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef __MACROS__ -#define __MACROS__ - -#define PI 3.1415926535897932384626433832795029f -#define PIx2 6.2831853071795864769252867665590058f - -#define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) -#define K_ELEMS_PER_GRID 2048 - -#define KERNEL_PHI_MAG_THREADS_PER_BLOCK 256 -#define KERNEL_Q_THREADS_PER_BLOCK 256 -#define KERNEL_Q_K_ELEMS_PER_GRID 1024 - -struct kValues { - float Kx; - float Ky; - float Kz; - float PhiMag; -}; - -#endif diff --git a/tests/opencl/mri-q/main.cc b/tests/opencl/mri-q/main.cc deleted file mode 100644 index 0f1a809b..00000000 --- a/tests/opencl/mri-q/main.cc +++ /dev/null @@ -1,321 +0,0 @@ -/*************************************************************************** - *cr - *cr (C) Copyright 2007 The Board of Trustees of the - *cr University of Illinois - *cr All Rights Reserved - *cr - ***************************************************************************/ - -/* - * C code for creating the Q data structure for fast convolution-based - * Hessian multiplication for arbitrary k-space trajectories. - * - * Inputs: - * kx - VECTOR of kx values, same length as ky and kz - * ky - VECTOR of ky values, same length as kx and kz - * kz - VECTOR of kz values, same length as kx and ky - * x - VECTOR of x values, same length as y and z - * y - VECTOR of y values, same length as x and z - * z - VECTOR of z values, same length as x and y - * phi - VECTOR of the Fourier transform of the spatial basis - * function, evaluated at [kx, ky, kz]. Same length as kx, ky, and kz. - * - * recommended g++ options: - * -O3 -lm -ffast-math -funroll-all-loops - */ - -#include -#include -#include -#include - -#include "ocl.h" -#include "file.h" -#include "macros.h" -#include "computeQ.h" - -static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) { - if (nullptr == filename || nullptr == data || 0 == size) - return CL_INVALID_VALUE; - - FILE* fp = fopen(filename, "r"); - if (NULL == fp) { - fprintf(stderr, "Failed to load kernel."); - return CL_INVALID_VALUE; - } - fseek(fp , 0 , SEEK_END); - long fsize = ftell(fp); - rewind(fp); - - *data = (uint8_t*)malloc(fsize); - *size = fread(*data, 1, fsize, fp); - - fclose(fp); - - return CL_SUCCESS; -} - -static void -setupMemoryGPU(int num, int size, cl_mem* dev_ptr, float* host_ptr,clPrmtr* clPrm) -{ - cl_int clStatus; - *dev_ptr = clCreateBuffer(clPrm->clContext,CL_MEM_READ_ONLY,num*size,NULL,&clStatus); - CHECK_ERROR("clCreateBuffer"); - clStatus = clEnqueueWriteBuffer(clPrm->clCommandQueue,*dev_ptr,CL_TRUE,0,num*size,host_ptr,0,NULL,NULL); - CHECK_ERROR("clEnequeueWriteBuffer"); -} - -static void -cleanupMemoryGPU(int num, int size, cl_mem* dev_ptr, float* host_ptr, clPrmtr* clPrm) -{ - cl_int clStatus; - clStatus = clEnqueueReadBuffer(clPrm->clCommandQueue,*dev_ptr,CL_TRUE,0,num*size,host_ptr,0,NULL,NULL); - CHECK_ERROR("clEnqueueReadBuffer") - clStatus = clReleaseMemObject(*dev_ptr); - CHECK_ERROR("clReleaseMemObject") -} - -int -main (int argc, char *argv[]) { - int numX, numK; /* Number of X and K values */ - int original_numK; /* Number of K values in input file */ - float *kx, *ky, *kz; /* K trajectory (3D vectors) */ - float *x, *y, *z; /* X coordinates (3D vectors) */ - float *phiR, *phiI; /* Phi values (complex) */ - float *phiMag; /* Magnitude of Phi */ - float *Qr, *Qi; /* Q signal (complex) */ - - struct kValues* kVals; - - struct pb_Parameters *params; - struct pb_TimerSet timers; - - pb_InitializeTimerSet(&timers); - - /* Read command line */ - params = pb_ReadParameters(&argc, argv); - - params->inpFiles = (char **)malloc(sizeof(char *) * 2); - params->inpFiles[0] = (char *)malloc(100); - params->inpFiles[1] = NULL; - strncpy(params->inpFiles[0], "32_32_32_dataset.bin", 100); - - if ((params->inpFiles[0] == NULL) || (params->inpFiles[1] != NULL)) - { - fprintf(stderr, "Expecting one input filename\n"); - exit(-1); - } - - /* Read in data */ - pb_SwitchToTimer(&timers, pb_TimerID_IO); - inputData(params->inpFiles[0], - &original_numK, &numX, - &kx, &ky, &kz, - &x, &y, &z, - &phiR, &phiI); - - /* Reduce the number of k-space samples if a number is given - * on the command line */ - if (argc < 2) - numK = original_numK; - else - { - int inputK; - char *end; - inputK = strtol(argv[1], &end, 10); - if (end == argv[1]) - { - fprintf(stderr, "Expecting an integer parameter\n"); - exit(-1); - } - - numK = MIN(inputK, original_numK); - } - - printf("%d pixels in output; %d samples in trajectory; using %d samples\n", - numX, original_numK, numK); - - pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); - - clPrmtr clPrm; - - pb_Context* pb_context; - pb_context = pb_InitOpenCLContext(params); - if (pb_context == NULL) { - fprintf (stderr, "Error: No OpenCL platform/device can be found."); - return -1; - } - - cl_int clStatus; - cl_device_id clDevice = (cl_device_id) pb_context->clDeviceId; - cl_platform_id clPlatform = (cl_platform_id) pb_context->clPlatformId; - clPrm.clContext = (cl_context) pb_context->clContext; - - clPrm.clCommandQueue = clCreateCommandQueue(clPrm.clContext,clDevice,CL_QUEUE_PROFILING_ENABLE,&clStatus); - CHECK_ERROR("clCreateCommandQueue") - - pb_SetOpenCL(&(clPrm.clContext), &(clPrm.clCommandQueue)); - -#ifdef HOSTGPU - const char* clSource[] = {readFile("kernel.cl")}; - CHECK_ERROR("clCreateProgramWithSource") - cl_program clProgram = clCreateProgramWithSource(clPrm.clContext,1,clSource,NULL,&clStatus); -#else - uint8_t *kernel_bin = NULL; - size_t kernel_size; - cl_int binary_status = 0; - CHECK_ERROR("read_kernel_file") - clStatus = read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size); - CHECK_ERROR("clCreateProgramWithSource") - cl_program clProgram = clCreateProgramWithBinary( - clPrm.clContext, 1, &clDevice, &kernel_size, (const uint8_t**)&kernel_bin, &binary_status, &clStatus); -#endif - - char options[50]; - sprintf(options,"-I src/opencl_nvidia"); - clStatus = clBuildProgram(clProgram,0,NULL,options,NULL,NULL); - if (clStatus != CL_SUCCESS) { - char buf[4096]; - clGetProgramBuildInfo(clProgram, clDevice, CL_PROGRAM_BUILD_LOG, 4096, buf, NULL); - printf ("%s\n", buf); - CHECK_ERROR("clBuildProgram") - } - - /* Create CPU data structures */ - createDataStructsCPU(numK, numX, &phiMag, &Qr, &Qi); - - /* GPU section 1 (precompute PhiMag) */ - { - clPrm.clKernel = clCreateKernel(clProgram,"ComputePhiMag_GPU",&clStatus); - CHECK_ERROR("clCreateKernel") - - /* Mirror several data structures on the device */ - cl_mem phiR_d; - cl_mem phiI_d; - cl_mem phiMag_d; - - pb_SwitchToTimer(&timers, pb_TimerID_COPY); - - setupMemoryGPU(numK,sizeof(float),&phiR_d,phiR,&clPrm); - setupMemoryGPU(numK,sizeof(float),&phiI_d,phiI,&clPrm); - phiMag_d = clCreateBuffer(clPrm.clContext,CL_MEM_WRITE_ONLY,numK*sizeof(float),NULL,&clStatus); - CHECK_ERROR("clCreateBuffer") - - clStatus = clFinish(clPrm.clCommandQueue); - CHECK_ERROR("clFinish") - - pb_SwitchToTimer(&timers, pb_TimerID_KERNEL); - - computePhiMag_GPU(numK, phiR_d, phiI_d, phiMag_d, &clPrm); - - clStatus = clFinish(clPrm.clCommandQueue); - CHECK_ERROR("clFinish") - - pb_SwitchToTimer(&timers, pb_TimerID_COPY); - - cleanupMemoryGPU(numK,sizeof(float),&phiMag_d,phiMag,&clPrm); - - clStatus = clReleaseMemObject(phiR_d); - CHECK_ERROR("clReleaseMemObject") - clStatus = clReleaseMemObject(phiI_d); - CHECK_ERROR("clReleaseMemObject") - } - - pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); - - kVals = (struct kValues*)calloc(numK, sizeof (struct kValues)); - - int k; - for (k = 0; k < numK; k++) { - kVals[k].Kx = kx[k]; - kVals[k].Ky = ky[k]; - kVals[k].Kz = kz[k]; - kVals[k].PhiMag = phiMag[k]; - } - - free(phiMag); - - clStatus = clReleaseKernel(clPrm.clKernel); - - /* GPU section 2 */ - { - clPrm.clKernel = clCreateKernel(clProgram,"ComputeQ_GPU",&clStatus); - CHECK_ERROR("clCreateKernel") - - cl_mem x_d; - cl_mem y_d; - cl_mem z_d; - cl_mem Qr_d; - cl_mem Qi_d; - - pb_SwitchToTimer(&timers, pb_TimerID_COPY); - - setupMemoryGPU(numX,sizeof(float),&x_d,x,&clPrm); - setupMemoryGPU(numX,sizeof(float),&y_d,y,&clPrm); - setupMemoryGPU(numX,sizeof(float),&z_d,z,&clPrm); - - Qr_d = clCreateBuffer(clPrm.clContext,CL_MEM_READ_WRITE,numX*sizeof(float),NULL,&clStatus); - CHECK_ERROR("clCreateBuffer") - clMemSet(&clPrm,Qr_d,0,numX*sizeof(float)); - Qi_d = clCreateBuffer(clPrm.clContext,CL_MEM_READ_WRITE,numX*sizeof(float),NULL,&clStatus); - CHECK_ERROR("clCreateBuffer") - clMemSet(&clPrm,Qi_d,0,numX*sizeof(float)); - - clStatus = clFinish(clPrm.clCommandQueue); - CHECK_ERROR("clFinish") - - pb_SwitchToTimer(&timers, pb_TimerID_KERNEL); - - computeQ_GPU(numK, numX, x_d, y_d, z_d, kVals, Qr_d, Qi_d, &clPrm); - - clStatus = clFinish(clPrm.clCommandQueue); - CHECK_ERROR("clFinish") - - pb_SwitchToTimer(&timers, pb_TimerID_COPY); - - clStatus = clReleaseMemObject(x_d); - CHECK_ERROR("clReleaseMemObject") - clStatus = clReleaseMemObject(y_d); - CHECK_ERROR("clReleaseMemObject") - clStatus = clReleaseMemObject(z_d); - CHECK_ERROR("clReleaseMemObject") - cleanupMemoryGPU(numX,sizeof(float),&Qr_d,Qr,&clPrm); - cleanupMemoryGPU(numX,sizeof(float),&Qi_d,Qi,&clPrm); - } - - pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); - - if (params->outFile) - { - /* Write Q to file */ - pb_SwitchToTimer(&timers, pb_TimerID_IO); - outputData(params->outFile, Qr, Qi, numX); - pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); - } - - free (kx); - free (ky); - free (kz); - free (x); - free (y); - free (z); - free (phiR); - free (phiI); - free (kVals); - free (Qr); - free (Qi); - - //free((void*)clSource[0]); - - clStatus = clReleaseKernel(clPrm.clKernel); - clStatus = clReleaseProgram(clProgram); - clStatus = clReleaseCommandQueue(clPrm.clCommandQueue); - clStatus = clReleaseContext(clPrm.clContext); - - pb_SwitchToTimer(&timers, pb_TimerID_NONE); - pb_PrintTimerSet(&timers); - - pb_FreeParameters(params); - - return 0; -} diff --git a/tests/opencl/mri-q/ocl copy.c b/tests/opencl/mri-q/ocl copy.c deleted file mode 100644 index 9ce9a2f5..00000000 --- a/tests/opencl/mri-q/ocl copy.c +++ /dev/null @@ -1,50 +0,0 @@ -#include -#include -#include -#include -#include "ocl.h" - -char* readFile(const char* fileName) -{ - FILE* fp; - fp = fopen(fileName,"r"); - if(fp == NULL) - { - printf("Error 1!\n"); - exit(1); - } - - fseek(fp,0,SEEK_END); - long size = ftell(fp); - rewind(fp); - - char* buffer = (char*)malloc(sizeof(char)*(size+1)); - if(buffer == NULL) - { - printf("Error 2!\n"); - fclose(fp); - exit(1); - } - - size_t res = fread(buffer,1,size,fp); - if(res != size) - { - printf("Error 3!\n"); - fclose(fp); - exit(1); - } - - buffer[size] = 0; - fclose(fp); - return buffer; -} - -void clMemSet(cl_command_queue clCommandQueue, cl_mem buf, int val, size_t size) -{ - cl_int clStatus; - char* temp = (char*)malloc(size); - memset(temp,val,size); - clStatus = clEnqueueWriteBuffer(clCommandQueue,buf,CL_TRUE,0,size,temp,0,NULL,NULL); - CHECK_ERROR("clEnqueueWriteBuffer") - free(temp); -} diff --git a/tests/opencl/mri-q/ocl copy.h b/tests/opencl/mri-q/ocl copy.h deleted file mode 100644 index 8840a868..00000000 --- a/tests/opencl/mri-q/ocl copy.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef __OCLH__ -#define __OCLH__ - -typedef struct { - cl_uint major; - cl_uint minor; - cl_uint multiProcessorCount; -} OpenCLDeviceProp; - -void clMemSet(cl_command_queue, cl_mem, int, size_t); -char* readFile(const char*); - -#define CHECK_ERROR(errorMessage) \ - if(clStatus != CL_SUCCESS) \ - { \ - printf("Error: %s!\n",errorMessage); \ - printf("Line: %d\n",__LINE__); \ - exit(1); \ - } - -#endif diff --git a/tests/opencl/mri-q/ocl.c b/tests/opencl/mri-q/ocl.c deleted file mode 100644 index 61cd5fe6..00000000 --- a/tests/opencl/mri-q/ocl.c +++ /dev/null @@ -1,50 +0,0 @@ -#include -#include -#include -#include "ocl.h" -#include - -char* readFile(const char* fileName) -{ - FILE* fp; - fp = fopen(fileName,"r"); - if(fp == NULL) - { - printf("Error 1!\n"); - exit(1); - } - - fseek(fp,0,SEEK_END); - long size = ftell(fp); - rewind(fp); - - char* buffer = (char*)malloc(sizeof(char)*(size+1)); - if(buffer == NULL) - { - printf("Error 2!\n"); - fclose(fp); - exit(1); - } - - size_t res = fread(buffer,1,size,fp); - if(res != size) - { - printf("Error 3!\n"); - fclose(fp); - exit(1); - } - - buffer[size] = 0; - fclose(fp); - return buffer; -} - -void clMemSet(clPrmtr* clPrm, cl_mem buf, int val, size_t size) -{ - cl_int clStatus; - char* temp = (char*)malloc(size); - memset(temp,val,size); - clStatus = clEnqueueWriteBuffer(clPrm->clCommandQueue,buf,CL_TRUE,0,size,temp,0,NULL,NULL); - CHECK_ERROR("clEnqueueWriteBuffer") - free(temp); -} diff --git a/tests/opencl/mri-q/ocl.h b/tests/opencl/mri-q/ocl.h deleted file mode 100644 index ce2876a7..00000000 --- a/tests/opencl/mri-q/ocl.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef __OCLH__ -#define __OCLH__ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - cl_context clContext; - cl_command_queue clCommandQueue; - cl_kernel clKernel; -} clPrmtr; - -void clMemSet(clPrmtr*, cl_mem, int, size_t); -char* readFile(const char*); - -#define CHECK_ERROR(errorMessage) \ - if(clStatus != CL_SUCCESS) \ - { \ - printf("Error: %s!\n",errorMessage); \ - printf("Line: %d\n",__LINE__); \ - exit(1); \ - } - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/tests/opencl/mri-q/parboil.h b/tests/opencl/mri-q/parboil.h deleted file mode 100644 index 4c9a8b5e..00000000 --- a/tests/opencl/mri-q/parboil.h +++ /dev/null @@ -1,348 +0,0 @@ -/* - * (c) 2010 The Board of Trustees of the University of Illinois. - */ -#ifndef PARBOIL_HEADER -#define PARBOIL_HEADER - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -/* A platform as specified by the user on the command line */ -struct pb_PlatformParam { - char *name; /* The platform name. This string is owned. */ - char *version; /* The platform version; may be NULL. - * This string is owned. */ -}; - -/* Create a PlatformParam from the given strings. - * 'name' must not be NULL. 'version' may be NULL. - * If not NULL, the strings should have been allocated by malloc(), - * and they will be owned by the returned object. - */ -struct pb_PlatformParam * -pb_PlatformParam(char *name, char *version); - -void -pb_FreePlatformParam(struct pb_PlatformParam *); - -/* A criterion for how to select a device */ -enum pb_DeviceSelectionCriterion { - pb_Device_INDEX, /* Enumerate the devices and select one - * by its number */ - pb_Device_CPU, /* Select a CPU device */ - pb_Device_GPU, /* Select a GPU device */ - pb_Device_ACCELERATOR, /* Select an accelerator device */ - pb_Device_NAME /* Select a device by name */ -}; - -/* A device as specified by the user on the command line */ -struct pb_DeviceParam { - enum pb_DeviceSelectionCriterion criterion; - union { - int index; /* If criterion == pb_Device_INDEX, - * the index of the device */ - char *name; /* If criterion == pb_Device_NAME, - * the name of the device. - * This string is owned. */ - }; -}; - -struct pb_DeviceParam * -pb_DeviceParam_index(int index); - -struct pb_DeviceParam * -pb_DeviceParam_cpu(void); - -struct pb_DeviceParam * -pb_DeviceParam_gpu(void); - -struct pb_DeviceParam * -pb_DeviceParam_accelerator(void); - -/* Create a by-name device selection criterion. - * The string should have been allocated by malloc(), and it will will be - * owned by the returned object. - */ -struct pb_DeviceParam * -pb_DeviceParam_name(char *name); - -void -pb_FreeDeviceParam(struct pb_DeviceParam *); - -/* Command line parameters for benchmarks */ -struct pb_Parameters { - char *outFile; /* If not NULL, the raw output of the - * computation should be saved to this - * file. The string is owned. */ - char **inpFiles; /* A NULL-terminated array of strings - * holding the input file(s) for the - * computation. The array and strings - * are owned. */ - struct pb_PlatformParam *platform; /* If not NULL, the platform - * specified on the command line. */ - struct pb_DeviceParam *device; /* If not NULL, the device - * specified on the command line. */ -}; - -/* Read command-line parameters. - * - * The argc and argv parameters to main are read, and any parameters - * interpreted by this function are removed from the argument list. - * - * A new instance of struct pb_Parameters is returned. - * If there is an error, then an error message is printed on stderr - * and NULL is returned. - */ -struct pb_Parameters * -pb_ReadParameters(int *_argc, char **argv); - -/* Free an instance of struct pb_Parameters. - */ -void -pb_FreeParameters(struct pb_Parameters *p); - -void -pb_FreeStringArray(char **); - -/* Count the number of input files in a pb_Parameters instance. - */ -int -pb_Parameters_CountInputs(struct pb_Parameters *p); - -/* A time or duration. */ -//#if _POSIX_VERSION >= 200112L -typedef unsigned long long pb_Timestamp; /* time in microseconds */ -//#else -//# error "Timestamps not implemented" -//#endif - -enum pb_TimerState { - pb_Timer_STOPPED, - pb_Timer_RUNNING, -}; - -struct pb_Timer { - enum pb_TimerState state; - pb_Timestamp elapsed; /* Amount of time elapsed so far */ - pb_Timestamp init; /* Beginning of the current time interval, - * if state is RUNNING. End of the last - * recorded time interfal otherwise. */ -}; - -/* Reset a timer. - * Use this to initialize a timer or to clear - * its elapsed time. The reset timer is stopped. - */ -void -pb_ResetTimer(struct pb_Timer *timer); - -/* Start a timer. The timer is set to RUNNING mode and - * time elapsed while the timer is running is added to - * the timer. - * The timer should not already be running. - */ -void -pb_StartTimer(struct pb_Timer *timer); - -/* Stop a timer. - * This stops adding elapsed time to the timer. - * The timer should not already be stopped. - */ -void -pb_StopTimer(struct pb_Timer *timer); - -/* Get the elapsed time in seconds. */ -double -pb_GetElapsedTime(struct pb_Timer *timer); - -/* Execution time is assigned to one of these categories. */ -enum pb_TimerID { - pb_TimerID_NONE = 0, - pb_TimerID_IO, /* Time spent in input/output */ - pb_TimerID_KERNEL, /* Time spent computing on the device, - * recorded asynchronously */ - pb_TimerID_COPY, /* Time spent synchronously moving data - * to/from device and allocating/freeing - * memory on the device */ - pb_TimerID_DRIVER, /* Time spent in the host interacting with the - * driver, primarily for recording the time - * spent queueing asynchronous operations */ - pb_TimerID_COPY_ASYNC, /* Time spent in asynchronous transfers */ - pb_TimerID_COMPUTE, /* Time for all program execution other - * than parsing command line arguments, - * I/O, kernel, and copy */ - pb_TimerID_OVERLAP, /* Time double-counted in asynchronous and - * host activity: automatically filled in, - * not intended for direct usage */ - pb_TimerID_LAST /* Number of timer IDs */ -}; - -/* Dynamic list of asynchronously tracked times between events */ -struct pb_async_time_marker_list { - char *label; // actually just a pointer to a string - enum pb_TimerID timerID; /* The ID to which the interval beginning - * with this marker should be attributed */ - void * marker; - //cudaEvent_t marker; /* The driver event for this marker */ - struct pb_async_time_marker_list *next; -}; - -struct pb_SubTimer { - char *label; - struct pb_Timer timer; - struct pb_SubTimer *next; -}; - -struct pb_SubTimerList { - struct pb_SubTimer *current; - struct pb_SubTimer *subtimer_list; -}; - -/* A set of timers for recording execution times. */ -struct pb_TimerSet { - enum pb_TimerID current; - struct pb_async_time_marker_list* async_markers; - pb_Timestamp async_begin; - pb_Timestamp wall_begin; - struct pb_Timer timers[pb_TimerID_LAST]; - struct pb_SubTimerList *sub_timer_list[pb_TimerID_LAST]; -}; - -/* Reset all timers in the set. */ -void -pb_InitializeTimerSet(struct pb_TimerSet *timers); - -void -pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category); - -/* Select which timer the next interval of time should be accounted - * to. The selected timer is started and other timers are stopped. - * Using pb_TimerID_NONE stops all timers. */ -void -pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer); - -void -pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID category); - -/* Print timer values to standard output. */ -void -pb_PrintTimerSet(struct pb_TimerSet *timers); - -/* Release timer resources */ -void -pb_DestroyTimerSet(struct pb_TimerSet * timers); - -void -pb_SetOpenCL(void *clContextPtr, void *clCommandQueuePtr); - - -typedef struct pb_Device_tag { - char* name; - void* clDevice; - int id; - unsigned int in_use; - unsigned int available; -} pb_Device; - -struct pb_Context_tag; -typedef struct pb_Context_tag pb_Context; - -typedef struct pb_Platform_tag { - char* name; - char* version; - void* clPlatform; - unsigned int in_use; - pb_Context** contexts; - pb_Device** devices; -} pb_Platform; - -struct pb_Context_tag { - void* clPlatformId; - void* clContext; - void* clDeviceId; - pb_Platform* pb_platform; - pb_Device* pb_device; -}; - -// verbosely print out list of platforms and their devices to the console. -pb_Platform** -pb_GetPlatforms(); - -// Choose a platform according to the given platform specification -pb_Platform* -pb_GetPlatform(struct pb_PlatformParam *platform); - -// choose a platform: by name, name & version -pb_Platform* -pb_GetPlatformByName(const char* name); - -pb_Platform* -pb_GetPlatformByNameAndVersion(const char* name, const char* version); - -// Choose a device according to the given device specification -pb_Device* -pb_GetDevice(pb_Platform* pb_platform, struct pb_DeviceParam *device); - -pb_Device** -pb_GetDevices(pb_Platform* pb_platform); - -// choose a device by name. -pb_Device* -pb_GetDeviceByName(pb_Platform* pb_platform, const char* name); - -pb_Platform* -pb_GetPlatformByEnvVars(); - -pb_Context* -pb_InitOpenCLContext(struct pb_Parameters* parameters); - -void -pb_ReleasePlatforms(); - -void -pb_ReleaseContext(pb_Context* c); - -void -pb_PrintPlatformInfo(pb_Context* c); - -void -perf_init(); - -//#define MEASURE_KERNEL_TIME - -#include - -#ifdef MEASURE_KERNEL_TIME -#define clEnqueueNDRangeKernel(q,k,d,o,dg,db,a,b,c) pb_clEnqueueNDRangeKernel((q), (k), (d), (o), (dg), (db), (a), (b), (c)) -cl_int -pb_clEnqueueNDRangeKernel(cl_command_queue /* command_queue */, - cl_kernel /* kernel */, - cl_uint /* work_dim */, - const size_t * /* global_work_offset */, - const size_t * /* global_work_size */, - const size_t * /* local_work_size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */); -#endif - -enum { T_FLOAT, T_DOUBLE, T_SHORT, T_INT, T_UCHAR }; -void pb_sig_float(char*, float*, int); -void pb_sig_double(char*, double*, int); -void pb_sig_short(char*, short*, int); -void pb_sig_int(char*, int*, int); -void pb_sig_uchar(char*, unsigned char*, unsigned int); -void pb_sig_clmem(char*, cl_command_queue, cl_mem, int); - -#ifdef __cplusplus -} -#endif - -#endif //PARBOIL_HEADER - diff --git a/tests/opencl/mri-q/parboil_opencl.c b/tests/opencl/mri-q/parboil_opencl.c deleted file mode 100644 index a4db1680..00000000 --- a/tests/opencl/mri-q/parboil_opencl.c +++ /dev/null @@ -1,1394 +0,0 @@ -/* - * (c) 2007 The Board of Trustees of the University of Illinois. - */ - -#include -#include -#include -#include -#include -#include - -#if _POSIX_VERSION >= 200112L -# include -#endif - -//#include "perfmon.h" - -cl_context *clContextPtr; -cl_command_queue *clCommandQueuePtr; - -// #define DISABLE_PARBOIL_TIMER - -/*****************************************************************************/ -/* Timer routines */ - -static int is_async(enum pb_TimerID timer) -{ - return (timer == pb_TimerID_KERNEL) || - (timer == pb_TimerID_COPY_ASYNC); -} - -static int is_blocking(enum pb_TimerID timer) -{ - return (timer == pb_TimerID_COPY) || (timer == pb_TimerID_NONE); -} - -#define INVALID_TIMERID pb_TimerID_LAST - -static int asyncs_outstanding(struct pb_TimerSet* timers) -{ - return (timers->async_markers != NULL) && - (timers->async_markers->timerID != INVALID_TIMERID); -} - -static struct pb_async_time_marker_list * -get_last_async(struct pb_TimerSet* timers) -{ - /* Find the last event recorded thus far */ - struct pb_async_time_marker_list * last_event = timers->async_markers; - if(last_event != NULL && last_event->timerID != INVALID_TIMERID) { - while(last_event->next != NULL && - last_event->next->timerID != INVALID_TIMERID) - last_event = last_event->next; - return last_event; - } else - return NULL; -} - -static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* - // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating Marker [%d]\n", timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = NULL; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - -static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = label; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - - -/* Assumes that all recorded events have completed */ -static pb_Timestamp record_async_times(struct pb_TimerSet* tset) -{ - struct pb_async_time_marker_list * next_interval = NULL; - struct pb_async_time_marker_list * last_marker = get_last_async(tset); - pb_Timestamp total_async_time = 0; - enum pb_TimerID timer; - - for(next_interval = tset->async_markers; next_interval != last_marker; - next_interval = next_interval->next) { - cl_ulong command_start=0, command_end=0; - cl_int ciErrNum = CL_SUCCESS; - - ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_start, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error getting first EventProfilingInfo: %d\n", ciErrNum); - } - - ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->next->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_end, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error getting second EventProfilingInfo: %d\n", ciErrNum); - } - - pb_Timestamp interval = (pb_Timestamp) (((double)(command_end - command_start)) / 1e3); - tset->timers[next_interval->timerID].elapsed += interval; - if (next_interval->label != NULL) { - struct pb_SubTimer *subtimer = tset->sub_timer_list[next_interval->timerID]->subtimer_list; - while (subtimer != NULL) { - if ( strcmp(subtimer->label, next_interval->label) == 0) { - subtimer->timer.elapsed += interval; - break; - } - subtimer = subtimer->next; - } - } - total_async_time += interval; - next_interval->timerID = INVALID_TIMERID; - } - - if(next_interval != NULL) - next_interval->timerID = INVALID_TIMERID; - - return total_async_time; -} - -static void -accumulate_time(pb_Timestamp *accum, - pb_Timestamp start, - pb_Timestamp end) -{ -//#if _POSIX_VERSION >= 200112L - *accum += end - start; -//#else -//# error "Timestamps not implemented for this system" -//#endif -} - -//#if _POSIX_VERSION >= 200112L -static pb_Timestamp get_time() -{ - //struct timeval tv; - //gettimeofday(&tv, NULL); - //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); - return 0; -} -//#else -//# error "no supported time libraries are available on this platform" -//#endif - -void -pb_ResetTimer(struct pb_Timer *timer) -{ -//#ifndef DISABLE_PARBOIL_TIMER - timer->state = pb_Timer_STOPPED; - -//#if _POSIX_VERSION >= 200112L - timer->elapsed = 0; -//#else -//# error "pb_ResetTimer: not implemented for this system" -//#endif -//#endif -} - -void -pb_StartTimer(struct pb_Timer *timer) -{ -/*#ifndef DISABLE_PARBOIL_TIMER - if (timer->state != pb_Timer_STOPPED) { - fputs("Ignoring attempt to start a running timer\n", stderr); - return; - } - - timer->state = pb_Timer_RUNNING; - -#if _POSIX_VERSION >= 200112L - { - struct timeval tv; - gettimeofday(&tv, NULL); - timer->init = tv.tv_sec * 1000000LL + tv.tv_usec; - } -#else -# error "pb_StartTimer: not implemented for this system" -#endif -#endif*/ -} - -void -pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) -{ -/*#ifndef DISABLE_PARBOIL_TIMER - - unsigned int numNotStopped = 0x3; // 11 - if (timer->state != pb_Timer_STOPPED) { - fputs("Warning: Timer was not stopped\n", stderr); - numNotStopped &= 0x1; // Zero out 2^1 - } - if (subtimer->state != pb_Timer_STOPPED) { - fputs("Warning: Subtimer was not stopped\n", stderr); - numNotStopped &= 0x2; // Zero out 2^0 - } - if (numNotStopped == 0x0) { - fputs("Ignoring attempt to start running timer and subtimer\n", stderr); - return; - } - - timer->state = pb_Timer_RUNNING; - subtimer->state = pb_Timer_RUNNING; - -#if _POSIX_VERSION >= 200112L - { - struct timeval tv; - gettimeofday(&tv, NULL); - - if (numNotStopped & 0x2) { - timer->init = tv.tv_sec * 1000000LL + tv.tv_usec; - } - - if (numNotStopped & 0x1) { - subtimer->init = tv.tv_sec * 1000000LL + tv.tv_usec; - } - } -#else -# error "pb_StartTimer: not implemented for this system" -#endif - -#endif*/ -} - -void -pb_StopTimer(struct pb_Timer *timer) -{ -/*#ifndef DISABLE_PARBOIL_TIMER - - pb_Timestamp fini; - - if (timer->state != pb_Timer_RUNNING) { - fputs("Ignoring attempt to stop a stopped timer\n", stderr); - return; - } - - timer->state = pb_Timer_STOPPED; - -#if _POSIX_VERSION >= 200112L - { - struct timeval tv; - gettimeofday(&tv, NULL); - fini = tv.tv_sec * 1000000LL + tv.tv_usec; - } -#else -# error "pb_StopTimer: not implemented for this system" -#endif - - accumulate_time(&timer->elapsed, timer->init, fini); - timer->init = fini; - -#endif*/ -} - -void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { -/*#ifndef DISABLE_PARBOIL_TIMER - - pb_Timestamp fini; - - unsigned int numNotRunning = 0x3; // 11 - if (timer->state != pb_Timer_RUNNING) { - fputs("Warning: Timer was not running\n", stderr); - numNotRunning &= 0x1; // Zero out 2^1 - } - if (subtimer->state != pb_Timer_RUNNING) { - fputs("Warning: Subtimer was not running\n", stderr); - numNotRunning &= 0x2; // Zero out 2^0 - } - if (numNotRunning == 0x0) { - fputs("Ignoring attempt to stop stopped timer and subtimer\n", stderr); - return; - } - - - timer->state = pb_Timer_STOPPED; - subtimer->state = pb_Timer_STOPPED; - -#if _POSIX_VERSION >= 200112L - { - struct timeval tv; - gettimeofday(&tv, NULL); - fini = tv.tv_sec * 1000000LL + tv.tv_usec; - } -#else -# error "pb_StopTimer: not implemented for this system" -#endif - - if (numNotRunning & 0x2) { - accumulate_time(&timer->elapsed, timer->init, fini); - timer->init = fini; - } - - if (numNotRunning & 0x1) { - accumulate_time(&subtimer->elapsed, subtimer->init, fini); - subtimer->init = fini; - } - -#endif*/ -} - -/* Get the elapsed time in seconds. */ -double -pb_GetElapsedTime(struct pb_Timer *timer) -{ - /*double ret; -#ifndef DISABLE_PARBOIL_TIMER - - if (timer->state != pb_Timer_STOPPED) { - fputs("Elapsed time from a running timer is inaccurate\n", stderr); - } - -#if _POSIX_VERSION >= 200112L - ret = timer->elapsed / 1e6; -#else -# error "pb_GetElapsedTime: not implemented for this system" -#endif -#endif - return ret;*/ - return 0; -} - -void -pb_InitializeTimerSet(struct pb_TimerSet *timers) -{ -/*#ifndef DISABLE_PARBOIL_TIMER - int n; - - timers->wall_begin = 0; //get_time(); - timers->current = pb_TimerID_NONE; - - timers->async_markers = NULL; - - for (n = 0; n < pb_TimerID_LAST; n++) { - pb_ResetTimer(&timers->timers[n]); - timers->sub_timer_list[n] = NULL; - } -#endif*/ -} - -void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { - clContextPtr = ((cl_context *)p_clContextPtr); - clCommandQueuePtr = ((cl_command_queue *)p_clCommandQueuePtr); -} - -void -pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { -/*#ifndef DISABLE_PARBOIL_TIMER - - struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc - (sizeof(struct pb_SubTimer)); - - int len = strlen(label); - - subtimer->label = (char *) malloc (sizeof(char)*(len+1)); - sprintf(subtimer->label, "%s\0", label); - - pb_ResetTimer(&subtimer->timer); - subtimer->next = NULL; - - struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[pb_Category]; - if (subtimerlist == NULL) { - subtimerlist = (struct pb_SubTimerList *) calloc - (1, sizeof(struct pb_SubTimerList)); - subtimerlist->subtimer_list = subtimer; - timers->sub_timer_list[pb_Category] = subtimerlist; - } else { - // Append to list - struct pb_SubTimer *element = subtimerlist->subtimer_list; - while (element->next != NULL) { - element = element->next; - } - element->next = subtimer; - } - -#endif*/ -} - -void -pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) -{ -#if 0 -#ifndef DISABLE_PARBOIL_TIMER - - /* Stop the currently running timer */ - if (timers->current != pb_TimerID_NONE) { - struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[timers->current]; - struct pb_SubTimer *currSubTimer = (subtimerlist != NULL) ? subtimerlist->current : NULL; - - if (!is_async(timers->current) ) { - if (timers->current != timer) { - if (currSubTimer != NULL) { - pb_StopTimerAndSubTimer(&timers->timers[timers->current], &currSubTimer->timer); - } else { - pb_StopTimer(&timers->timers[timers->current]); - } - } else { - if (currSubTimer != NULL) { - pb_StopTimer(&currSubTimer->timer); - } - } - } else { - insert_marker(timers, timer); - if (!is_async(timer)) { // if switching to async too, keep driver going - pb_StopTimer(&timers->timers[pb_TimerID_DRIVER]); - } - } - } - - pb_Timestamp currentTime = 0; //get_time(); - - /* The only cases we check for asynchronous task completion is - * when an overlapping CPU operation completes, or the next - * segment blocks on completion of previous async operations */ - if( asyncs_outstanding(timers) && - (!is_async(timers->current) || is_blocking(timer) ) ) { - - struct pb_async_time_marker_list * last_event = get_last_async(timers); - /* CL_COMPLETE if completed */ - - cl_int ciErrNum = CL_SUCCESS; - cl_int async_done = CL_COMPLETE; - - ciErrNum = clGetEventInfo(*((cl_event *)last_event->marker), CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof(cl_int), &async_done, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Querying EventInfo!\n"); - } - - - if(is_blocking(timer)) { - /* Async operations completed after previous CPU operations: - * overlapped time is the total CPU time since this set of async - * operations were first issued */ - - // timer to switch to is COPY or NONE - if(async_done != CL_COMPLETE) { - accumulate_time(&(timers->timers[pb_TimerID_OVERLAP].elapsed), - timers->async_begin,currentTime); - } - - /* Wait on async operation completion */ - ciErrNum = clWaitForEvents(1, (cl_event *)last_event->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Waiting for Events!\n"); - } - - pb_Timestamp total_async_time = record_async_times(timers); - - /* Async operations completed before previous CPU operations: - * overlapped time is the total async time */ - if(async_done == CL_COMPLETE) { - //fprintf(stderr, "Async_done: total_async_type = %lld\n", total_async_time); - timers->timers[pb_TimerID_OVERLAP].elapsed += total_async_time; - } - - } else - /* implies (!is_async(timers->current) && asyncs_outstanding(timers)) */ - // i.e. Current Not Async (not KERNEL/COPY_ASYNC) but there are outstanding - // so something is deeper in stack - if(async_done == CL_COMPLETE ) { - /* Async operations completed before previous CPU operations: - * overlapped time is the total async time */ - timers->timers[pb_TimerID_OVERLAP].elapsed += record_async_times(timers); - } - } - - /* Start the new timer */ - if (timer != pb_TimerID_NONE) { - if(!is_async(timer)) { - pb_StartTimer(&timers->timers[timer]); - } else { - // toSwitchTo Is Async (KERNEL/COPY_ASYNC) - if (!asyncs_outstanding(timers)) { - /* No asyncs outstanding, insert a fresh async marker */ - - insert_marker(timers, timer); - timers->async_begin = currentTime; - } else if(!is_async(timers->current)) { - /* Previous asyncs still in flight, but a previous SwitchTo - * already marked the end of the most recent async operation, - * so we can rename that marker as the beginning of this async - * operation */ - - struct pb_async_time_marker_list * last_event = get_last_async(timers); - last_event->label = NULL; - last_event->timerID = timer; - } - if (!is_async(timers->current)) { - pb_StartTimer(&timers->timers[pb_TimerID_DRIVER]); - } - } - } - timers->current = timer; - -#endif -#endif -} - -void -pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID category) -{ -#if 0 -#ifndef DISABLE_PARBOIL_TIMER - struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[timers->current]; - struct pb_SubTimer *curr = (subtimerlist != NULL) ? subtimerlist->current : NULL; - - if (timers->current != pb_TimerID_NONE) { - if (!is_async(timers->current) ) { - if (timers->current != category) { - if (curr != NULL) { - pb_StopTimerAndSubTimer(&timers->timers[timers->current], &curr->timer); - } else { - pb_StopTimer(&timers->timers[timers->current]); - } - } else { - if (curr != NULL) { - pb_StopTimer(&curr->timer); - } - } - } else { - insert_submarker(timers, label, category); - if (!is_async(category)) { // if switching to async too, keep driver going - pb_StopTimer(&timers->timers[pb_TimerID_DRIVER]); - } - } - } - - pb_Timestamp currentTime = 0; //get_time(); - - /* The only cases we check for asynchronous task completion is - * when an overlapping CPU operation completes, or the next - * segment blocks on completion of previous async operations */ - if( asyncs_outstanding(timers) && - (!is_async(timers->current) || is_blocking(category) ) ) { - - struct pb_async_time_marker_list * last_event = get_last_async(timers); - /* CL_COMPLETE if completed */ - - cl_int ciErrNum = CL_SUCCESS; - cl_int async_done = CL_COMPLETE; - - ciErrNum = clGetEventInfo(*((cl_event *)last_event->marker), CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof(cl_int), &async_done, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Querying EventInfo!\n"); - } - - if(is_blocking(category)) { - /* Async operations completed after previous CPU operations: - * overlapped time is the total CPU time since this set of async - * operations were first issued */ - - // timer to switch to is COPY or NONE - // if it hasn't already finished, then just take now and use that as the elapsed time in OVERLAP - // anything happening after now isn't OVERLAP because everything is being stopped to wait for synchronization - // it seems that the extra sync wall time isn't being recorded anywhere - if(async_done != CL_COMPLETE) - accumulate_time(&(timers->timers[pb_TimerID_OVERLAP].elapsed), - timers->async_begin,currentTime); - - /* Wait on async operation completion */ - ciErrNum = clWaitForEvents(1, (cl_event *)last_event->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Waiting for Events!\n"); - } - pb_Timestamp total_async_time = record_async_times(timers); - - /* Async operations completed before previous CPU operations: - * overlapped time is the total async time */ - // If it did finish, then accumulate all the async time that did happen into OVERLAP - // the immediately preceding EventSynchronize theoretically didn't have any effect since it was already completed. - if(async_done == CL_COMPLETE /*cudaSuccess*/) - timers->timers[pb_TimerID_OVERLAP].elapsed += total_async_time; - - } else - /* implies (!is_async(timers->current) && asyncs_outstanding(timers)) */ - // i.e. Current Not Async (not KERNEL/COPY_ASYNC) but there are outstanding - // so something is deeper in stack - if(async_done == CL_COMPLETE /*cudaSuccess*/) { - /* Async operations completed before previous CPU operations: - * overlapped time is the total async time */ - timers->timers[pb_TimerID_OVERLAP].elapsed += record_async_times(timers); - } - // else, this isn't blocking, so just check the next time around - } - - subtimerlist = timers->sub_timer_list[category]; - struct pb_SubTimer *subtimer = NULL; - - if (label != NULL) { - subtimer = subtimerlist->subtimer_list; - while (subtimer != NULL) { - if (strcmp(subtimer->label, label) == 0) { - break; - } else { - subtimer = subtimer->next; - } - } - } - - /* Start the new timer */ - if (category != pb_TimerID_NONE) { - if(!is_async(category)) { - if (subtimerlist != NULL) { - subtimerlist->current = subtimer; - } - - if (category != timers->current && subtimer != NULL) { - pb_StartTimerAndSubTimer(&timers->timers[category], &subtimer->timer); - } else if (subtimer != NULL) { - pb_StartTimer(&subtimer->timer); - } else { - pb_StartTimer(&timers->timers[category]); - } - } else { - if (subtimerlist != NULL) { - subtimerlist->current = subtimer; - } - - // toSwitchTo Is Async (KERNEL/COPY_ASYNC) - if (!asyncs_outstanding(timers)) { - /* No asyncs outstanding, insert a fresh async marker */ - insert_submarker(timers, label, category); - timers->async_begin = currentTime; - } else if(!is_async(timers->current)) { - /* Previous asyncs still in flight, but a previous SwitchTo - * already marked the end of the most recent async operation, - * so we can rename that marker as the beginning of this async - * operation */ - - struct pb_async_time_marker_list * last_event = get_last_async(timers); - last_event->timerID = category; - last_event->label = label; - } // else, marker for switchToThis was already inserted - - //toSwitchto is already asynchronous, but if current/prev state is async too, then DRIVER is already running - if (!is_async(timers->current)) { - pb_StartTimer(&timers->timers[pb_TimerID_DRIVER]); - } - } - } - - timers->current = category; -#endif -#endif -} - -void -pb_PrintTimerSet(struct pb_TimerSet *timers) -{ -#if 0 -#ifndef DISABLE_PARBOIL_TIMER - pb_Timestamp wall_end = 0; //get_time(); - - struct pb_Timer *t = timers->timers; - struct pb_SubTimer* sub = NULL; - - int maxSubLength; - - const char *categories[] = { - "IO", "Kernel", "Copy", "Driver", "Copy Async", "Compute" - }; - - const int maxCategoryLength = 10; - - int i; - for(i = 1; i < pb_TimerID_LAST-1; ++i) { // exclude NONE and OVRELAP from this format - if(pb_GetElapsedTime(&t[i]) != 0) { - - // Print Category Timer - printf("%-*s: %f\n", maxCategoryLength, categories[i-1], pb_GetElapsedTime(&t[i])); - - if (timers->sub_timer_list[i] != NULL) { - sub = timers->sub_timer_list[i]->subtimer_list; - maxSubLength = 0; - while (sub != NULL) { - // Find longest SubTimer label - if (strlen(sub->label) > maxSubLength) { - maxSubLength = strlen(sub->label); - } - sub = sub->next; - } - - // Fit to Categories - if (maxSubLength <= maxCategoryLength) { - maxSubLength = maxCategoryLength; - } - - sub = timers->sub_timer_list[i]->subtimer_list; - - // Print SubTimers - while (sub != NULL) { - printf(" -%-*s: %f\n", maxSubLength, sub->label, pb_GetElapsedTime(&sub->timer)); - sub = sub->next; - } - } - } - } - - if(pb_GetElapsedTime(&t[pb_TimerID_OVERLAP]) != 0) - printf("CPU/Kernel Overlap: %f\n", pb_GetElapsedTime(&t[pb_TimerID_OVERLAP])); - - float walltime = (wall_end - timers->wall_begin)/ 1e6; - printf("Timer Wall Time: %f\n", walltime); - -#endif -#endif -} - -void pb_DestroyTimerSet(struct pb_TimerSet * timers) -{ -#ifndef DISABLE_PARBOIL_TIMER - /* clean up all of the async event markers */ - struct pb_async_time_marker_list* event = timers->async_markers; - while(event != NULL) { - - cl_int ciErrNum = CL_SUCCESS; - ciErrNum = clWaitForEvents(1, (cl_event *)(event)->marker); - if (ciErrNum != CL_SUCCESS) { - //fprintf(stderr, "Error Waiting for Events!\n"); - } - - ciErrNum = clReleaseEvent( *((cl_event *)(event)->marker) ); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Release Events!\n"); - } - - free((event)->marker); - struct pb_async_time_marker_list* next = ((event)->next); - - free(event); - - // (*event) = NULL; - event = next; - } - - int i = 0; - for(i = 0; i < pb_TimerID_LAST; ++i) { - if (timers->sub_timer_list[i] != NULL) { - struct pb_SubTimer *subtimer = timers->sub_timer_list[i]->subtimer_list; - struct pb_SubTimer *prev = NULL; - while (subtimer != NULL) { - free(subtimer->label); - prev = subtimer; - subtimer = subtimer->next; - free(prev); - } - free(timers->sub_timer_list[i]); - } - } -#endif -} - -static pb_Platform** ptr = NULL; - -// verbosely print out list of platforms and their devices to the console. -pb_Platform** -pb_GetPlatforms() { - if (ptr == NULL) { - cl_uint num_platforms; - clGetPlatformIDs(0, NULL, &num_platforms); - if (num_platforms == 0) return NULL; - - ptr = (pb_Platform **) malloc(sizeof(pb_Platform *) * (num_platforms + 1)); - cl_platform_id* ids = (cl_platform_id *) malloc(num_platforms * sizeof(cl_platform_id)); - clGetPlatformIDs(num_platforms, ids, NULL); - - unsigned int i; - for (i = 0; i < num_platforms; i++) { - ptr[i] = (pb_Platform *) malloc(sizeof(pb_Platform)); - ptr[i]->clPlatform = ids[i]; - ptr[i]->contexts = NULL; - ptr[i]->in_use = 0; - ptr[i]->devices = NULL; - - size_t sz; - clGetPlatformInfo(ids[i], CL_PLATFORM_NAME, 0, NULL, &sz); - char* name = (char *) malloc(sz + 1); - clGetPlatformInfo(ids[i], CL_PLATFORM_NAME, sz, name, NULL); - name[sz] = '\0'; - ptr[i]->name = name; - - clGetPlatformInfo(ids[i], CL_PLATFORM_VERSION, 0, NULL, &sz); - char* version = (char *) malloc(sz + 1); - clGetPlatformInfo(ids[i], CL_PLATFORM_VERSION, sz, version, NULL); - version[sz] = '\0'; - ptr[i]->version = version; - } - ptr[i] = NULL; - - free(ids); - } - - return (pb_Platform**) ptr; -} - -pb_Context* -createContext(pb_Platform* pb_platform, pb_Device* pb_device) { - pb_Context* c = (pb_Context*) malloc(sizeof(pb_Context)); - cl_int clStatus; - cl_context_properties clCps[3] = { - CL_CONTEXT_PLATFORM, (cl_context_properties)(pb_platform->clPlatform), 0 - }; - c->clContext = - clCreateContext(clCps, 1, (cl_device_id*)&pb_device->clDevice, NULL, NULL, &clStatus); - c->clPlatformId = pb_platform->clPlatform; - c->clDeviceId = pb_device->clDevice; - c->pb_platform = pb_platform; - c->pb_device = pb_device; - pb_platform->in_use = 1; - pb_device->in_use = 1; - unsigned int i = 0; - if (pb_platform->contexts == NULL) { - pb_platform->contexts = (pb_Context**) malloc(2*sizeof(pb_Context*)); - } else { - for (i = 0; pb_platform->contexts[i] != NULL; i++) {}; - pb_platform->contexts = (pb_Context**) realloc(pb_platform->contexts, - (i+1)*sizeof(pb_Context*)); - } - pb_platform->contexts[i+1] = NULL; - pb_platform->contexts[i] = c; - return c; -} - -// choose a platform by name. -pb_Platform* -pb_GetPlatformByName(const char* name) { - pb_Platform** ps = (pb_Platform **) pb_GetPlatforms(); - if (ps == NULL) return NULL; - if (name == NULL) { - return *ps; - } - - while (*ps) { - if (strstr((*ps)->name, name)) break; - ps++; - } - return (pb_Platform*) *ps; -} - -pb_Device** -pb_GetDevices(pb_Platform* pb_platform) { - if (pb_platform->devices == NULL) { - cl_uint num_devs; - cl_device_id* dev_ids; - clGetDeviceIDs((cl_platform_id) pb_platform->clPlatform, - CL_DEVICE_TYPE_ALL, 0, NULL, &num_devs); - if (num_devs == 0) return NULL; - - pb_platform->devices = - (pb_Device **) malloc((num_devs + 1) * sizeof(pb_Device *)); - dev_ids = (cl_device_id *) malloc(sizeof(cl_device_id) * num_devs); - clGetDeviceIDs((cl_platform_id) pb_platform->clPlatform, - CL_DEVICE_TYPE_ALL, num_devs, dev_ids, NULL); - - unsigned int i; - for (i = 0; i < num_devs; i++) { - pb_platform->devices[i] = (pb_Device *) malloc(sizeof(pb_Device)); - - pb_platform->devices[i]->clDevice = dev_ids[i]; - pb_platform->devices[i]->id = i; - - size_t sz; - clGetDeviceInfo(dev_ids[i], CL_DEVICE_NAME, 0, NULL, &sz); - char* name = (char *) malloc(sz + 1); - clGetDeviceInfo(dev_ids[i], CL_DEVICE_NAME, sz, name, NULL); - name[sz] = '\0'; - pb_platform->devices[i]->name = (char *) name; - - cl_bool available; - clGetDeviceInfo(dev_ids[i], CL_DEVICE_AVAILABLE, sizeof(cl_bool), &available, NULL); - pb_platform->devices[i]->available = (int) available; - - pb_platform->devices[i]->in_use = 0; - } - pb_platform->devices[i] = NULL; - } - return (pb_Device **) pb_platform->devices; -} - -// choose a device by name. -static pb_Device* -pb_SelectDeviceByName(pb_Device **ds, const char* name) { - if (ds == NULL) return NULL; - if (name == NULL) return *ds; - while (*ds) { - if (strstr((*ds)->name, name)) break; - ds++; - } - - return *ds; -} - -// choose a device by name and set the device's 'in_use' flag. -pb_Device* -pb_GetDeviceByName(pb_Platform* pb_platform, const char* name) { - pb_Device** ds = (pb_Device **) pb_GetDevices(pb_platform); - pb_Device *d = pb_SelectDeviceByName(ds, name); - - if (d) d->in_use = 1; - - return d; -} - -void -pb_ReleasePlatforms() { - if (!ptr) return; - pb_Platform** cur_ptr = ptr; - while (*cur_ptr) { - pb_Platform* pfptr = *cur_ptr++; - if (pfptr->devices) { - pb_Device** dvptr = pfptr->devices; - while (*dvptr) { - pb_Device* d = *dvptr++; - free(d->name); - free(d); - } - free(pfptr->devices); - } - if (pfptr->contexts) { - pb_Context** cptr = pfptr->contexts; - while (*cptr) { - free(*cptr++); - } - free(pfptr->contexts); - } - free(pfptr->name); - free(pfptr); - } - free(ptr); - ptr = NULL; -} - -pb_Platform* -pb_GetPlatformByNameAndVersion(const char* name, const char* version) { - pb_Platform** ps = (pb_Platform **) pb_GetPlatforms(); - if (ps == NULL) return NULL; - if (name == NULL) return *ps; - while (*ps) { - if (strstr((*ps)->name, name) && strstr((*ps)->version, version)) break; - ps++; - } - return (pb_Platform*) *ps; -} - -/* Return a pointer to the device at the specified index, or NULL. - * Used by pb_GetDevice. */ -static pb_Device * -select_device_by_index(pb_Device** ds, int id) -{ - int i = 0; - pb_Device** p = ds; - while (*p && (i < id)) { p++; i++; } - return *p; -} - -/* Return a pointer to the device with the specified type, or NULL. - * Used by pb_GetDevice. */ -static pb_Device * -select_device_by_type(pb_Device** ds, - enum pb_DeviceSelectionCriterion criterion) -{ - cl_device_type sought_type; - - /* Determine the OpenCL device type to search for */ - switch(criterion) { - case pb_Device_CPU: - sought_type = CL_DEVICE_TYPE_CPU; - break; - case pb_Device_GPU: - sought_type = CL_DEVICE_TYPE_GPU; - break; - case pb_Device_ACCELERATOR: - sought_type = CL_DEVICE_TYPE_ACCELERATOR; - break; - default: - fprintf(stderr, "pb_GetDevice: Invalid device type"); - exit(-1); - } - - /* Find the device */ - { - pb_Device** p = ds; - cl_device_type type; - while (*p) { - clGetDeviceInfo(((cl_device_id) ((*p)->clDevice)), CL_DEVICE_TYPE, - sizeof(cl_device_type), &type, NULL); - if (type == sought_type) break; - } - - return *p; - } -} - -pb_Device* -pb_GetDevice(pb_Platform* pb_platform, struct pb_DeviceParam *device) -{ - pb_Device** ds = (pb_Device **) pb_GetDevices(pb_platform); - - // The list of devices must be nonempty - if (ds == NULL || *ds == NULL) { - fprintf(stderr, "Error: No device is found in platform: name = %s, version = %s\n.", pb_platform->name, pb_platform->version); - exit(-1); - } - - pb_Device *selected_device = NULL; - - if (device != NULL) { - /* Use 'device' to select and return a device. - * If unable to select a device, fall - * back on the default selection mechanism. */ - switch(device->criterion) { - case pb_Device_INDEX: - selected_device = select_device_by_index(ds, device->index); - break; - case pb_Device_GPU: - case pb_Device_CPU: - case pb_Device_ACCELERATOR: - selected_device = select_device_by_type(ds, device->criterion); - break; - case pb_Device_NAME: - selected_device = pb_SelectDeviceByName(ds, device->name); - break; - default: - fprintf(stderr, "pb_GetDevice: Invalid argument"); - exit(-1); - } - } - - /* By default or if user-specified selection failed, - * select the first device */ - if (selected_device == NULL) - selected_device = *ds; - - /* Set the in_use flag */ - selected_device->in_use = 1; - - return selected_device; -} - -pb_Device* -pb_GetDeviceByEnvVars(pb_Platform* pb_platform) { - - /* Convert environment variables to a 'pb_DeviceParam' */ - struct pb_DeviceParam *param = NULL; - - char* device_num = getenv("PARBOIL_DEVICE_NUMBER"); - if (device_num && strcmp(device_num, "")) { - int id = atoi(device_num); - param = pb_DeviceParam_index(id); - } - else { - char* device_name = getenv("PARBOIL_DEVICE_NAME"); - if (device_name && strcmp(device_name, "")) { - param = pb_DeviceParam_name(strdup(device_name)); - } - else { - char* device_type = getenv("PARBOIL_DEVICE_TYPE"); - if (device_type && strcmp(device_type, "")) { - if (strcmp(device_type, "CPU") == 0) - param = pb_DeviceParam_cpu(); - else if (strcmp(device_type, "GPU") == 0) - param = pb_DeviceParam_gpu(); - else if (strcmp(device_type, "ACCELERATOR") == 0) - param = pb_DeviceParam_accelerator(); - } - } - } - - /* Get a device */ - pb_Device *d = pb_GetDevice(pb_platform, param); - pb_FreeDeviceParam(param); - - return d; -} - -pb_Platform* -pb_GetPlatformByEnvVars() { - char* name = getenv("PARBOIL_PLATFORM_NAME"); - char* version = getenv("PARBOIL_PLATFORM_VERSION"); - - /* Create a pb_PlatformParam object (or NULL) representing the data from the - * environment variables */ - struct pb_PlatformParam *platform; - - if (name) { - if (version) { - platform = pb_PlatformParam(strdup(name), strdup(version)); - } - else { - platform = pb_PlatformParam(strdup(name), NULL); - } - } - else { - platform = NULL; - } - - /* Convert to a platform */ - pb_Platform *p = pb_GetPlatform(platform); - pb_FreePlatformParam(platform); - - return p; -} - -/* Choose an OpenCL platform based on the given command-line parameters. - * If NULL, use the default OpenCL platform. */ -pb_Platform* -pb_GetPlatform(struct pb_PlatformParam *platform) { - if (platform != NULL) { - /* Try to use command-line parameters to choose platform */ - char *name = platform->name; - char *version = platform->version; - - if (!name) { - fprintf(stderr, "Internal error: NULL pointer"); - exit(-1); - } - - if (version) { - pb_Platform* p = pb_GetPlatformByNameAndVersion(name, version); - if (p) return p; - } - - pb_Platform* p = pb_GetPlatformByName(name); - if (p) return p; - } - - pb_Platform* p = pb_GetPlatformByName(NULL); - if (p == NULL) { - fprintf(stderr, "Error: No OpenCL platform in this system. Exiting."); - exit(-1); - } - return p; -} - -//extern void perf_init(); -//extern void mxpa_scheduler_init(); - -pb_Context* -pb_InitOpenCLContext(struct pb_Parameters* parameters) { -#if 0 - pb_Platform* ps = pb_GetPlatform(parameters->platform); - if (!ps) return NULL; - pb_Device* ds = pb_GetDevice(ps, parameters->device); - if (!ds) return NULL; - - /* HERE INITIALIZE TIMER */ - //perf_init(); - //mxpa_scheduler_init(); - - pb_Context* c = createContext(ps, ds); - pb_PrintPlatformInfo(c); - return c; -#endif - cl_int _err; - cl_platform_id platform_id; - cl_device_id device_id; - cl_context context; - clGetPlatformIDs(1, &platform_id, NULL); - clGetDeviceIDs(platform_id, CL_DEVICE_TYPE_DEFAULT, 1, &device_id, NULL); - context = clCreateContext(NULL, 1, &device_id, NULL, NULL, &_err); - - pb_Context* c = (pb_Context*)malloc(sizeof(pb_Context)); - c->clContext = context; - c->clDeviceId = device_id; - c->clPlatformId = platform_id; - c->pb_platform = (pb_Platform*)malloc(sizeof(pb_Platform)); - c->pb_device = (pb_Device*)malloc(sizeof(pb_Device)); - c->pb_platform->devices = (pb_Device**)malloc(sizeof(pb_Device*) * 2); - c->pb_platform->devices[0] = c->pb_device; - c->pb_platform->devices[1] = NULL; - c->pb_platform->contexts = (pb_Context**)malloc(sizeof(pb_Context*) * 2); - c->pb_platform->contexts[0] = c; - c->pb_platform->contexts[1] = NULL; - c->pb_platform->in_use = 1; - c->pb_device->in_use = 1; - return c; -} - -void -pb_ReleaseOpenCLContext(pb_Context* c) { - pb_ReleasePlatforms(); -} - -void -pb_PrintPlatformInfo(pb_Context* c) { - /*pb_Platform** ps = pb_GetPlatforms(); - if (!ps) { - fprintf (stderr, "No platform found"); - return; - } - - printf ("********************************************************\n"); - printf ("DETECTED OPENCL PLATFORMS AND DEVICES:\n"); - printf ("--------------------------------------------------------\n"); - - while (*ps) { - printf ("PLATFORM = %s, %s", (*ps)->name, (*ps)->version); - if (c->pb_platform == *ps) printf (" (SELECTED)"); - printf ("\n"); - - pb_Device** ds = (pb_Device **) pb_GetDevices((*ps)); - if (ds == NULL) { - printf (" + (No devices)\n"); - } else { - while (*ds) { - printf (" + %d: %s", (*ds)->id, (*ds)->name); - if (c->pb_device == *ds) printf (" (SELECTED)"); - printf ("\n"); - ds++; - } - } - - ps++; - } - printf ("********************************************************\n");*/ -} - -#ifdef MEASURE_KERNEL_TIME - -#undef clEnqueueNDRangeKernel - -//extern void pin_trace_enable(char*); -//extern void pin_trace_disable(char*); - -cl_int -pb_clEnqueueNDRangeKernel(cl_command_queue q/* command_queue */, - cl_kernel k/* kernel */, - cl_uint d/* work_dim */, - const size_t * o/* global_work_offset */, - const size_t * gws/* global_work_size */, - const size_t * lws/* local_work_size */, - cl_uint n/* num_events_in_wait_list */, - const cl_event * w/* event_wait_list */, - cl_event * e/* event */) { - - char buf[128]; - struct timeval begin, end; - clGetKernelInfo(k, CL_KERNEL_FUNCTION_NAME, 128, buf, NULL); - -#if 0 - int i; - for (i = 0; i < d; i++) { - printf ("%s: %d: %d / %d\n", buf, i, gws[i], (lws == NULL ? 0 : lws[i])); - } -#endif - - clFinish(q); clFlush(q); - //pin_trace_enable(buf); - //gettimeofday(&begin, NULL); - cl_int result = clEnqueueNDRangeKernel(q, k, d, o, gws, lws, n, w, e); - clFinish(q); clFlush(q); - //gettimeofday(&end, NULL); - //pin_trace_disable(buf); - //float t = (float)(end.tv_sec - begin.tv_sec) + (end.tv_usec - begin.tv_usec) / 1000000.0f; - fflush(stdout); - fflush(stderr); - //printf ("PBTIMER: %s: %f\n", buf, t); - return result; -} - -#endif - -void -pb_sig_float(char* c, float* p, int sz) { - int i; - double s = 0.0; - for (i = 0; i < sz; i++) s += p[i] * (float)(i+1); - printf ("[Signature] %s = %lf\n", c, s); -} - -void -pb_sig_double(char* c, double* p, int sz) { - int i; - double s = 0.0; - for (i = 0; i < sz; i++) s += p[i]; - printf ("[Signature] %s = %lf\n", c, s); -} - -void -pb_sig_short(char* c, short* p, int sz) { - int i; - long long int s = 0; - for (i = 0; i < sz; i++) s += p[i]; - printf ("[Signature] %s = %lld\n", c, s); -} - -void -pb_sig_int(char* c, int* p, int sz) { - int i; - long long int s = 0; - for (i = 0; i < sz; i++) s += p[i]; - printf ("[Signature] %s = %lld\n", c, s); -} - -void -pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { - int i; - unsigned long long int s = 0; - for (i = 0; i < sz; i++) s += p[i]; - printf ("[Signature] %s = %lld\n", c, s); -} - -void pb_sig_clmem(char* s, cl_command_queue command_queue, cl_mem memobj, int ty) { - size_t sz; - if (clGetMemObjectInfo(memobj, CL_MEM_SIZE, sizeof(size_t), &sz, NULL) != CL_SUCCESS) { - printf ("Something wrong.\n"); - assert(0); - } else { - printf ("size = %d\n", sz); - } - char* hp; // = (char*) malloc(sz); - //posix_memalign((void**)&hp, 64, sz); - hp = (char*)malloc(sz); - - clEnqueueReadBuffer (command_queue, - memobj, - CL_TRUE, - 0, - sz, - hp, - 0, - NULL, - NULL); - - if (ty == T_FLOAT) pb_sig_float(s, (float*)hp, sz/sizeof(float)); - if (ty == T_DOUBLE) pb_sig_double(s, (double*)hp, sz/sizeof(double)); - if (ty == T_INT) pb_sig_int(s, (int*)hp, sz/sizeof(int)); - if (ty == T_SHORT) pb_sig_short(s, (short*)hp, sz/sizeof(short)); - if (ty == T_UCHAR) pb_sig_uchar(s, (unsigned char*)hp, sz/sizeof(char)); - - free(hp); -} - diff --git a/tests/opencl/oclprintf/main.cc b/tests/opencl/oclprintf/main.cc index 7c0463cf..4af39802 100644 --- a/tests/opencl/oclprintf/main.cc +++ b/tests/opencl/oclprintf/main.cc @@ -106,11 +106,6 @@ int main (int argc, char **argv) { cl_platform_id platform_id; size_t kernel_size; - cl_int binary_status; - - // read kernel binary from file - if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) - return -1; // Getting platform and device information CL_CHECK(clGetPlatformIDs(1, &platform_id, NULL)); @@ -124,12 +119,17 @@ int main (int argc, char **argv) { a_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_READ_ONLY, nbytes, NULL, &_err)); printf("Create program from kernel source\n"); - program = CL_CHECK2(clCreateProgramWithBinary( - context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, &binary_status, &_err)); - if (program == NULL) { - cleanup(); +#ifdef HOSTGPU + if (0 != read_kernel_file("kernel.cl", &kernel_bin, &kernel_size)) return -1; - } + program = CL_CHECK2(clCreateProgramWithSource( + context, 1, (const char**)&kernel_bin, &kernel_size, &_err)); +#else + if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) + return -1; + program = CL_CHECK2(clCreateProgramWithBinary( + context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, NULL, &_err)); +#endif // Build program CL_CHECK(clBuildProgram(program, 1, &device_id, NULL, NULL, NULL)); @@ -143,7 +143,7 @@ int main (int argc, char **argv) { // Allocate memories for input arrays and output arrays. h_a = (int*)malloc(nbytes); - // Initialize values for array members. + // Generate input values for (int i = 0; i < size; ++i) { h_a[i] = -1 + i; } diff --git a/tests/opencl/psort/main.cc b/tests/opencl/psort/main.cc index 26a42807..8bc834dc 100644 --- a/tests/opencl/psort/main.cc +++ b/tests/opencl/psort/main.cc @@ -115,11 +115,6 @@ int main (int argc, char **argv) { cl_platform_id platform_id; size_t kernel_size; - cl_int binary_status; - - // read kernel binary from file - if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) - return -1; // Getting platform and device information CL_CHECK(clGetPlatformIDs(1, &platform_id, NULL)); @@ -134,12 +129,17 @@ int main (int argc, char **argv) { c_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_WRITE_ONLY, nbytes, NULL, &_err)); printf("Create program from kernel source\n"); - program = CL_CHECK2(clCreateProgramWithBinary( - context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, &binary_status, &_err)); - if (program == NULL) { - cleanup(); +#ifdef HOSTGPU + if (0 != read_kernel_file("kernel.cl", &kernel_bin, &kernel_size)) return -1; - } + program = CL_CHECK2(clCreateProgramWithSource( + context, 1, (const char**)&kernel_bin, &kernel_size, &_err)); +#else + if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) + return -1; + program = CL_CHECK2(clCreateProgramWithBinary( + context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, NULL, &_err)); +#endif // Build program CL_CHECK(clBuildProgram(program, 1, &device_id, NULL, NULL, NULL)); @@ -155,17 +155,16 @@ int main (int argc, char **argv) { h_a = (int*)malloc(nbytes); h_c = (int*)malloc(nbytes); - // Initialize values for array members. + // Generate input values for (int i = 0; i < size; ++i) { - h_c[i] = 0xdeadbeef; if (float_enable) { float value = sinf(i)*sinf(i); - h_a[i] = *(int*)&value; - printf("*** [%d]: h_a=%f\n", i, value); + ((float*)h_a)[i] = value; + printf("*** [%d]: %f\n", i, value); } else { int value = size*sinf(i); h_a[i] = value; - printf("*** [%d]: h_a=%d\n", i, value); + printf("*** [%d]: %d\n", i, value); } } @@ -189,38 +188,44 @@ int main (int argc, char **argv) { CL_CHECK(clEnqueueReadBuffer(commandQueue, c_memobj, CL_TRUE, 0, nbytes, h_c, 0, NULL, NULL)); printf("Verify result\n"); - for (int i = 0; i < size; ++i) { - int value = h_c[i]; + for (int i = 0; i < size; ++i) { if (float_enable) { - printf("*** [%d]: h_a=%f\n", i, *(float*)&value); + float value = ((float*)h_c)[i]; + printf("*** [%d]: %f\n", i, value); } else { - printf("*** [%d]: h_a=%d\n", i, value); + int value = h_c[i]; + printf("*** [%d]: %d\n", i, value); } } int errors = 0; - for (int i = 0; i < size; ++i) { - int ref = h_a[i]; - float ref_f = *(float*)&ref; + for (int i = 0; i < size; ++i) { int pos = 0; - for (int j = 0; j < size; ++j) { - int cur = h_a[j]; - if (float_enable) { - float cur_f = *(float*)&cur; - pos += (cur_f < ref_f) || (cur_f == ref_f && j < i); - } else { + if (float_enable) { + float ref = ((float*)h_a)[i]; + for (int j = 0; j < size; ++j) { + float cur = ((float*)h_a)[j]; + pos += (cur < ref) || (cur == ref && j < i); + } + float value = ((float*)h_c)[pos]; + if (value != ref) { + if (errors < 100) { + printf("*** error: [%d] expected=%f, actual=%f\n", pos, ref, value); + } + ++errors; + } + } else { + int ref = h_a[i]; + for (int j = 0; j < size; ++j) { + int cur = h_a[j]; pos += (cur < ref) || (cur == ref && j < i); } - } - int value = h_c[pos]; - if (value != ref) { - if (errors < 100) { - if (float_enable) { - printf("*** error: [%d] expected=%f, actual=%f\n", pos, ref_f, *(float*)&value); - } else { + int value = h_c[pos]; + if (value != ref) { + if (errors < 100) { printf("*** error: [%d] expected=%d, actual=%d\n", pos, ref, value); } + ++errors; } - ++errors; } } if (0 == errors) { diff --git a/tests/opencl/saxpy/main.cc b/tests/opencl/saxpy/main.cc index 4ea15759..9355c945 100644 --- a/tests/opencl/saxpy/main.cc +++ b/tests/opencl/saxpy/main.cc @@ -151,16 +151,12 @@ int main(int argc, char **argv) { cl_platform_id platform_id; cl_device_id device_id; + cl_program program; cl_mem input_buffer; cl_mem output_buffer; size_t kernel_size; cl_context context; cl_command_queue queue; - cl_int binary_status = 0; - - // read kernel binary from file - if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) - return -1; // Getting platform and device information CL_CHECK(clGetPlatformIDs(1, &platform_id, NULL)); @@ -172,19 +168,18 @@ int main(int argc, char **argv) { cl_kernel kernel = 0; cl_mem memObjects[2] = {0, 0}; - // Create OpenCL program - first attempt to load cached binary. - // If that is not available, then create the program from source - // and store the binary for future use. - printf("create program from binary...\n"); - cl_program program = CL_CHECK_ERR(clCreateProgramWithBinary( - context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, &binary_status, &_err)); - if (program == NULL) { - std::cerr << "Failed to write program binary" << std::endl; - Cleanup(device_id, context, queue, program, kernel, memObjects); - return 1; - } else { - printf("Read program from binary.\n"); - } + printf("Create program from kernel source\n"); +#ifdef HOSTGPU + if (0 != read_kernel_file("kernel.cl", &kernel_bin, &kernel_size)) + return -1; + program = CL_CHECK_ERR(clCreateProgramWithSource( + context, 1, (const char**)&kernel_bin, &kernel_size, &_err)); +#else + if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) + return -1; + program = CL_CHECK_ERR(clCreateProgramWithBinary( + context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, NULL, &_err)); +#endif // Build program CL_CHECK(clBuildProgram(program, 1, &device_id, NULL, NULL, NULL)); diff --git a/tests/opencl/sfilter/main.cc b/tests/opencl/sfilter/main.cc index 3a7a5979..0ae264a5 100644 --- a/tests/opencl/sfilter/main.cc +++ b/tests/opencl/sfilter/main.cc @@ -149,14 +149,10 @@ int main(int argc, char **argv) { cl_platform_id platform_id; cl_device_id device_id; + cl_program program; size_t kernel_size; - cl_int binary_status = 0; uint8_t *kernel_bin = NULL; - // read kernel binary from file - if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) - return -1; - // Getting platform and device information CL_CHECK(clGetPlatformIDs(1, &platform_id, NULL)); CL_CHECK(clGetDeviceIDs(platform_id, CL_DEVICE_TYPE_DEFAULT, 1, &device_id, NULL)); @@ -170,19 +166,18 @@ int main(int argc, char **argv) { cl_kernel kernel = 0; cl_mem memObjects[2] = {0, 0}; - // Create OpenCL program - first attempt to load cached binary. - // If that is not available, then create the program from source - // and store the binary for future use. - printf("create program from binary...\n"); - cl_program program = CL_CHECK_ERR(clCreateProgramWithBinary( - context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, &binary_status, &_err)); - if (program == NULL) { - std::cerr << "Failed to write program binary" << std::endl; - Cleanup(kernel_bin, device_id, context, queue, program, kernel, memObjects); - return 1; - } else { - printf("Read program from binary."); - } + printf("Create program from kernel source\n"); +#ifdef HOSTGPU + if (0 != read_kernel_file("kernel.cl", &kernel_bin, &kernel_size)) + return -1; + program = CL_CHECK_ERR(clCreateProgramWithSource( + context, 1, (const char**)&kernel_bin, &kernel_size, &_err)); +#else + if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) + return -1; + program = CL_CHECK_ERR(clCreateProgramWithBinary( + context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, NULL, &_err)); +#endif // Build program CL_CHECK(clBuildProgram(program, 1, &device_id, NULL, NULL, NULL)); diff --git a/tests/opencl/sgemm/common.h b/tests/opencl/sgemm/common.h index 01f68d48..fdb40bce 100644 --- a/tests/opencl/sgemm/common.h +++ b/tests/opencl/sgemm/common.h @@ -1,12 +1,8 @@ #ifndef COMMON_H #define COMMON_H -#define USE_FLOAT - -#ifdef USE_FLOAT +#ifndef TYPE #define TYPE float -#else -#define TYPE int #endif #endif // COMMON_H \ No newline at end of file diff --git a/tests/opencl/sgemm/main.cc b/tests/opencl/sgemm/main.cc index 3ca14792..bc48dff0 100644 --- a/tests/opencl/sgemm/main.cc +++ b/tests/opencl/sgemm/main.cc @@ -11,6 +11,8 @@ #define KERNEL_NAME "sgemm" +#define FLOAT_ULP 6 + #define CL_CHECK(_expr) \ do { \ cl_int _err = _expr; \ @@ -33,6 +35,66 @@ _ret; \ }) +template +class Comparator {}; + +template <> +class Comparator { +public: + static const char* type_str() { + return "integer"; + } + static int generate() { + return rand(); + } + static bool compare(int a, int b, int index, int errors) { + if (a != b) { + if (errors < 100) { + printf("*** error: [%d] expected=%d, actual=%d\n", index, a, b); + } + return false; + } + return true; + } +}; + +template <> +class Comparator { +public: + static const char* type_str() { + return "float"; + } + static int generate() { + return static_cast(rand()) / RAND_MAX; + } + static bool compare(float a, float b, int index, int errors) { + union fi_t { float f; int32_t i; }; + fi_t fa, fb; + fa.f = a; + fb.f = b; + auto d = std::abs(fa.i - fb.i); + if (d > FLOAT_ULP) { + if (errors < 100) { + printf("*** error: [%d] expected=%f, actual=%f\n", index, a, b); + } + return false; + } + return true; + } +}; + +/*static void sgemm_cpu(TYPE *C, const TYPE* A, const TYPE *B, int M, int N, int K) { + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + TYPE acc = 0; + for (int k = 0; k < K; ++k) { + acc += A[k * M + m] * B[n * K + k]; + } + C[n * M + m] = acc; + } + } +}*/ + static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) { if (nullptr == filename || nullptr == data || 0 == size) return -1; @@ -54,32 +116,6 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) return 0; } -/*static void matmul(TYPE *C, const TYPE* A, const TYPE *B, int M, int N, int K) { - for (int m = 0; m < M; ++m) { - for (int n = 0; n < N; ++n) { - TYPE acc = 0; - for (int k = 0; k < K; ++k) { - acc += A[k * M + m] * B[n * K + k]; - } - C[n * M + m] = acc; - } - } -}*/ - -#ifdef USE_FLOAT -static bool compare_equal(float a, float b, int ulp = 21) { - union fi_t { int i; float f; }; - fi_t fa, fb; - fa.f = a; - fb.f = b; - return std::abs(fa.i - fb.i) <= ulp; -} -#else -static bool compare_equal(int a, int b, int ulp = 21) { - return (a == b); -} -#endif - cl_device_id device_id = NULL; cl_context context = NULL; cl_command_queue commandQueue = NULL; @@ -145,15 +181,12 @@ int main (int argc, char **argv) { // parse command arguments parse_args(argc, argv); + uint32_t num_points = size * size; + cl_platform_id platform_id; size_t kernel_size; - cl_int binary_status; srand(50); - - // read kernel binary from file - if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) - return -1; // Getting platform and device information CL_CHECK(clGetPlatformIDs(1, &platform_id, NULL)); @@ -163,18 +196,23 @@ int main (int argc, char **argv) { context = CL_CHECK2(clCreateContext(NULL, 1, &device_id, NULL, NULL, &_err)); // Allocate device buffers - size_t nbytes = size * size * sizeof(TYPE); + size_t nbytes = num_points * sizeof(TYPE); a_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_READ_ONLY, nbytes, NULL, &_err)); b_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_READ_ONLY, nbytes, NULL, &_err)); c_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_WRITE_ONLY, nbytes, NULL, &_err)); printf("Create program from kernel source\n"); - program = CL_CHECK2(clCreateProgramWithBinary( - context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, &binary_status, &_err)); - if (program == NULL) { - cleanup(); +#ifdef HOSTGPU + if (0 != read_kernel_file("kernel.cl", &kernel_bin, &kernel_size)) return -1; - } + program = CL_CHECK2(clCreateProgramWithSource( + context, 1, (const char**)&kernel_bin, &kernel_size, &_err)); +#else + if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) + return -1; + program = CL_CHECK2(clCreateProgramWithBinary( + context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, NULL, &_err)); +#endif // Build program CL_CHECK(clBuildProgram(program, 1, &device_id, NULL, NULL, NULL)); @@ -194,23 +232,17 @@ int main (int argc, char **argv) { h_b = (TYPE*)malloc(nbytes); h_c = (TYPE*)malloc(nbytes); - // Initialize values for array members. - for (int i = 0; i < (size * size); ++i) { - #ifdef USE_FLOAT - h_a[i] = (float)rand() / (float)RAND_MAX; - h_b[i] = (float)rand() / (float)RAND_MAX; - #else - h_a[i] = rand(); - h_b[i] = rand(); - #endif - h_c[i] = 0xdeadbeef; + // Generate input values + for (uint32_t i = 0; i < num_points; ++i) { + h_a[i] = Comparator::generate(); + h_b[i] = Comparator::generate(); } size_t global_offset[2] = {0, 0}; size_t global_work_size[2] = {size, size}; size_t local_work_size[2] = {1, 1}; - std::vector ref_vec(size * size); + std::vector ref_vec(num_points); // reference generation size_t num_groups_y = global_work_size[1] / local_work_size[1]; @@ -228,12 +260,7 @@ int main (int argc, char **argv) { TYPE acc = 0; for (int k = 0; k < width; k++) { acc += h_a[k * width + r] * h_b[c * width + k]; - } - /*#ifdef USE_FLOAT - printf("*** r=%d, c=%d, v=%f\n", r, c, acc); - #else - printf("*** r=%d, c=%d, v=%d\n", r, c, acc); - #endif*/ + } ref_vec[c * width + r] = acc; } } @@ -260,14 +287,8 @@ int main (int argc, char **argv) { printf("Verify result\n"); int errors = 0; - for (int i = 0; i < (size * size); i++) { - if (!compare_equal(h_c[i], ref_vec[i])) { - if (errors < 100) - #ifdef USE_FLOAT - printf("*** error: [%d] expected=%f, actual=%f\n", i, ref_vec[i], h_c[i]); - #else - printf("*** error: [%d] expected=%d, actual=%d\n", i, ref_vec[i], h_c[i]); - #endif + for (uint32_t i = 0; i < num_points; ++i) { + if (!Comparator::compare(h_c[i], ref_vec[i], i, errors)) { ++errors; } } diff --git a/tests/opencl/spmv/convert_dataset.c b/tests/opencl/spmv/convert_dataset.c index 122d8819..aba9c3b3 100644 --- a/tests/opencl/spmv/convert_dataset.c +++ b/tests/opencl/spmv/convert_dataset.c @@ -91,15 +91,11 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, if ((f = fopen(mtx_filename, "r")) == NULL) exit(1); - printf("OK**\n"); - if (mm_read_banner(f, &matcode) != 0) { printf("Could not process Matrix Market banner.\n"); exit(1); } - printf("OK**\n"); - /* This is how one can screen matrix types if their application */ /* only supports a subset of the Matrix Market data types. */ diff --git a/tests/opencl/spmv/main.cc b/tests/opencl/spmv/main.cc index 85182322..01aa43cd 100644 --- a/tests/opencl/spmv/main.cc +++ b/tests/opencl/spmv/main.cc @@ -148,7 +148,6 @@ int main(int argc, char **argv) { // &h_data, &h_indices, &h_ptr, // &h_perm, &h_nzcnt); int col_count; - printf("OK--\n"); coo_to_jds(parameters->inpFiles[0], // bcsstk32.mtx, fidapm05.mtx, jgl009.mtx 1, // row padding pad, // warp size @@ -159,8 +158,6 @@ int main(int argc, char **argv) { &h_data, &h_ptr, &h_nzcnt, &h_indices, &h_perm, &col_count, &dim, &len, &nzcnt_len, &depth); - printf("OK++\n"); - // pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); h_Ax_vector = (float *)malloc(sizeof(float) * dim); h_x_vector = (float *)malloc(sizeof(float) * dim); diff --git a/tests/opencl/stencil/main.cc b/tests/opencl/stencil/main.cc index a68bd5a3..cbbed6bc 100644 --- a/tests/opencl/stencil/main.cc +++ b/tests/opencl/stencil/main.cc @@ -157,9 +157,7 @@ int main(int argc, char** argv) { CHECK_ERROR("clBuildProgram") cl_kernel clKernel = clCreateKernel(clProgram,"naive_kernel",&clStatus); - CHECK_ERROR("clCreateKernel") - - printf("OK+\n"); + CHECK_ERROR("clCreateKernel") //host data float *h_A0; @@ -177,15 +175,11 @@ int main(int argc, char** argv) { h_Anext=(float*)malloc(sizeof(float)*size); pb_SwitchToTimer(&timers, pb_TimerID_IO); //FILE *fp = fopen(parameters->inpFiles[0], "rb"); - printf("OK+\n"); read_data(h_A0, nx,ny,nz,NULL); - printf("OK+\n"); - //fclose(fp); - memcpy (h_Anext,h_A0,sizeof(float)*size); + //fclose(fp); + memcpy (h_Anext,h_A0,sizeof(float)*size); pb_SwitchToTimer(&timers, pb_TimerID_COPY); - - printf("OK+\n"); //memory allocation d_A0 = clCreateBuffer(clContext,CL_MEM_READ_WRITE,size*sizeof(float),NULL,&clStatus); @@ -201,18 +195,16 @@ int main(int argc, char** argv) { pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); - printf("OK+\n"); - //only use 1D thread block - int tx = 128; + int tx = 128; size_t block[3] = {tx,1,1}; size_t grid[3] = {(nx-2+tx-1)/tx*tx,ny-2,nz-2}; - //size_t grid[3] = {nx-2,ny-2,nz-2}; - size_t offset[3] = {1,1,1}; - printf("grid size in x/y/z = %d %d %d\n",grid[0],grid[1],grid[2]); + //size_t grid[3] = {nx-2,ny-2,nz-2}; + size_t offset[3] = {1,1,1}; + printf("grid size in x/y/z = %d %d %d\n",grid[0],grid[1],grid[2]); printf("block size in x/y/z = %d %d %d\n",block[0],block[1],block[2]); - printf ("blocks = %d\n", (grid[0]/block[0])*(grid[1]/block[1])*(grid[2]*block[2])); + printf ("blocks = %d\n", (grid[0]/block[0])*(grid[1]/block[1])*(grid[2]*block[2])); clStatus = clSetKernelArg(clKernel,0,sizeof(float),(void*)&c0); clStatus = clSetKernelArg(clKernel,1,sizeof(float),(void*)&c1); @@ -226,14 +218,10 @@ int main(int argc, char** argv) { //main execution pb_SwitchToTimer(&timers, pb_TimerID_KERNEL); - printf("OK+0\n"); - int t; for(t=0;toutFile) { pb_SwitchToTimer(&timers, pb_TimerID_IO); diff --git a/tests/opencl/vecadd/main.cc b/tests/opencl/vecadd/main.cc index 23aa49b4..e443f7c5 100644 --- a/tests/opencl/vecadd/main.cc +++ b/tests/opencl/vecadd/main.cc @@ -122,11 +122,6 @@ int main (int argc, char **argv) { cl_platform_id platform_id; size_t kernel_size; - cl_int binary_status; - - // read kernel binary from file - if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) - return -1; // Getting platform and device information CL_CHECK(clGetPlatformIDs(1, &platform_id, NULL)); @@ -142,13 +137,17 @@ int main (int argc, char **argv) { c_memobj = CL_CHECK2(clCreateBuffer(context, CL_MEM_WRITE_ONLY, nbytes, NULL, &_err)); printf("Create program from kernel source\n"); - cl_int _err; - program = clCreateProgramWithBinary( - context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, &binary_status, &_err); - if (program == NULL) { - cleanup(); +#ifdef HOSTGPU + if (0 != read_kernel_file("kernel.cl", &kernel_bin, &kernel_size)) return -1; - } + program = CL_CHECK2(clCreateProgramWithSource( + context, 1, (const char**)&kernel_bin, &kernel_size, &_err)); +#else + if (0 != read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size)) + return -1; + program = CL_CHECK2(clCreateProgramWithBinary( + context, 1, &device_id, &kernel_size, (const uint8_t**)&kernel_bin, NULL, &_err)); +#endif // Build program CL_CHECK(clBuildProgram(program, 1, &device_id, NULL, NULL, NULL)); @@ -166,12 +165,10 @@ int main (int argc, char **argv) { h_b = (float*)malloc(nbytes); h_c = (float*)malloc(nbytes); - // Initialize values for array members. + // Generate input values for (int i = 0; i < size; ++i) { h_a[i] = sinf(i)*sinf(i); h_b[i] = cosf(i)*cosf(i); - h_c[i] = 0xdeadbeef; - //printf("*** [%d]: h_a=%f, h_b=%f\n", i, h_a[i], h_b[i]); } // Creating command queue diff --git a/tests/opencl/vectorhypot/Makefile b/tests/opencl/vectorhypot/Makefile deleted file mode 100644 index 5763dfb5..00000000 --- a/tests/opencl/vectorhypot/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -PROJECT = vectorhypot - -SRCS = main.cc oclUtils.cpp shrUtils.cpp cmd_arg_reader.cpp - -CXXFLAGS += -I. - -OPTS ?= - -include ../common.mk - diff --git a/tests/opencl/vectorhypot/cmd_arg_reader.cpp b/tests/opencl/vectorhypot/cmd_arg_reader.cpp deleted file mode 100644 index 9b7f91ef..00000000 --- a/tests/opencl/vectorhypot/cmd_arg_reader.cpp +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright 1993-2010 NVIDIA Corporation. All rights reserved. - * - * Please refer to the NVIDIA end user license agreement (EULA) associated - * with this source code for terms and conditions that govern your use of - * this software. Any use, reproduction, disclosure, or distribution of - * this software and related documentation outside the terms of the EULA - * is strictly prohibited. - * - */ - -/* CUda UTility Library */ - -// includes, file -#include "cmd_arg_reader.h" - -// includes, system -#include - -// internal unnamed namespace - -namespace -{ - // types, internal (class, enum, struct, union, typedef) - - // variables, internal - -} // namespace { - -// variables, exported - -/*static*/ CmdArgReader* CmdArgReader::self; -/*static*/ char** CmdArgReader::rargv; -/*static*/ int CmdArgReader::rargc; - -// functions, exported - -//////////////////////////////////////////////////////////////////////////////// -//! Public construction interface -//! @return a handle to the class instance -//! @param argc number of command line arguments (as given to main()) -//! @param argv command line argument string (as given to main()) -//////////////////////////////////////////////////////////////////////////////// -/*static*/ void -CmdArgReader::init( const int argc, const char** argv) -{ - if ( NULL != self) - { - return; - } - - // command line arguments - if (( 0 == argc) || ( 0 == argv)) - { - LOGIC_EXCEPTION( "No command line arguments given."); - } - - self = new CmdArgReader(); - - self->createArgsMaps( argc, argv); - - rargc = argc; - rargv = const_cast( argv); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Constructor, default -//////////////////////////////////////////////////////////////////////////////// -CmdArgReader::CmdArgReader() : - args(), - unprocessed(), - iter(), - iter_unprocessed() -{ } - -//////////////////////////////////////////////////////////////////////////////// -//! Destructor -//////////////////////////////////////////////////////////////////////////////// -CmdArgReader::~CmdArgReader() -{ - for( iter = args.begin(); iter != args.end(); ++iter) - { - if( *(iter->second.first) == typeid( int)) - { - delete static_cast( iter->second.second); - break; - } - else if( *(iter->second.first) == typeid( bool)) - { - delete static_cast( iter->second.second); - break; - } - else if( *(iter->second.first) == typeid( std::string)) - { - delete static_cast( iter->second.second); - break; - } - else if( *(iter->second.first) == typeid( std::vector< std::string>) ) - { - delete static_cast< std::vector< std::string>* >( iter->second.second); - break; - } - else if( *(iter->second.first) == typeid( std::vector) ) - { - delete static_cast< std::vector* >( iter->second.second); - break; - } - } -} - -//////////////////////////////////////////////////////////////////////////////// -//! Read args as token value pair into map for better processing (Even the -//! values remain strings until the parameter values is requested by the -//! program.) -//! @param argc the argument count (as given to 'main') -//! @param argv the char* array containing the command line arguments -//////////////////////////////////////////////////////////////////////////////// -void -CmdArgReader::createArgsMaps( const int argc, const char** argv) { - - std::string token; - std::string val_str; - - std::map< std::string, std::string> args; - - std::string::size_type pos; - std::string arg; - for( int i=1; i - inline const T* getArgHelper( const std::string& name); - - //! Check if a command line argument with name \a name exists - //! @return true if a command line argument of name \a name exists, - //! otherwise false - //! @param name the name of the requested argument - inline bool existArgHelper( const std::string& name) const; - - //! Read args as token value pair into map for better processing - //! (Even the values remain strings until the parameter values is - //! requested by the program.) - //! @param argc the argument count (as given to 'main') - //! @param argv the char* array containing the command line arguments - void createArgsMaps( const int argc, const char** argv); - - //! Helper for "casting" the strings from the map with the unprocessed - //! values to the correct - //! data type. - //! @return true if conversion succeeded, otherwise false - //! @param element the value as string - //! @param val the value as type T - template - static inline bool convertToT( const std::string& element, T& val); - -public: - - // typedefs internal - - //! container for a processed command line argument - //! typeid is used to easily be able to decide if a re-requested token-value - //! pair match the type of the first conversion - typedef std::pair< const std::type_info*, void*> ValType; - //! map of already converted values - typedef std::map< std::string, ValType > ArgsMap; - //! iterator for the map of already converted values - typedef ArgsMap::iterator ArgsMapIter; - typedef ArgsMap::const_iterator ConstArgsMapIter; - - //! map of unprocessed (means unconverted) token-value pairs - typedef std::map< std::string, std::string> UnpMap; - //! iterator for the map of unprocessed (means unconverted) token-value pairs - typedef std::map< std::string, std::string>::iterator UnpMapIter; - -private: - -#ifdef _WIN32 -# pragma warning( disable: 4251) -#endif - - //! rargc original value of argc - static int rargc; - - //! rargv contains command line arguments in raw format - static char** rargv; - - //! args Map containing the already converted token-value pairs - ArgsMap args; - - //! args Map containing the unprocessed / unconverted token-value pairs - UnpMap unprocessed; - - //! iter Iterator for the map with the already converted token-value - //! pairs (to avoid frequent reallocation) - ArgsMapIter iter; - - //! iter Iterator for the map with the unconverted token-value - //! pairs (to avoid frequent reallocation) - UnpMapIter iter_unprocessed; - -#ifdef _WIN32 -# pragma warning( default: 4251) -#endif - -private: - - //! Constructor, copy (not implemented) - CmdArgReader( const CmdArgReader&); - - //! Assignment operator (not implemented) - CmdArgReader& operator=( const CmdArgReader&); -}; - -// variables, exported (extern) - -// functions, inlined (inline) - -//////////////////////////////////////////////////////////////////////////////// -//! Conversion function for command line argument arrays -//! @note This function is used each type for which no template specialization -//! exist (which will cause errors if the type does not fulfill the std::vector -//! interface). -//////////////////////////////////////////////////////////////////////////////// -template -/*static*/ inline bool -CmdArgReader::convertToT( const std::string& element, T& val) -{ - // preallocate storage - val.resize( std::count( element.begin(), element.end(), ',') + 1); - - unsigned int i = 0; - std::string::size_type pos_start = 1; // leave array prefix '[' - std::string::size_type pos_end = 0; - - // do for all elements of the comma seperated list - while( std::string::npos != ( pos_end = element.find(',', pos_end+1)) ) - { - // convert each element by the appropriate function - if ( ! convertToT< typename T::value_type >( - std::string( element, pos_start, pos_end - pos_start), val[i])) - { - return false; - } - - pos_start = pos_end + 1; - ++i; - } - - std::string tmp1( element, pos_start, element.length() - pos_start - 1); - - // process last element (leave array postfix ']') - if ( ! convertToT< typename T::value_type >( std::string( element, - pos_start, - element.length() - pos_start - 1), - val[i])) - { - return false; - } - - // possible to process all elements? - return true; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Conversion function for command line arguments of type int -//////////////////////////////////////////////////////////////////////////////// -template<> -inline bool -CmdArgReader::convertToT( const std::string& element, int& val) -{ - std::istringstream ios( element); - ios >> val; - - bool ret_val = false; - if ( ios.eof()) - { - ret_val = true; - } - - return ret_val; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Conversion function for command line arguments of type float -//////////////////////////////////////////////////////////////////////////////// -template<> -inline bool -CmdArgReader::convertToT( const std::string& element, float& val) -{ - std::istringstream ios( element); - ios >> val; - - bool ret_val = false; - if ( ios.eof()) - { - ret_val = true; - } - - return ret_val; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Conversion function for command line arguments of type double -//////////////////////////////////////////////////////////////////////////////// -template<> -inline bool -CmdArgReader::convertToT( const std::string& element, double& val) -{ - std::istringstream ios( element); - ios >> val; - - bool ret_val = false; - if ( ios.eof()) - { - ret_val = true; - } - - return ret_val; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Conversion function for command line arguments of type string -//////////////////////////////////////////////////////////////////////////////// -template<> -inline bool -CmdArgReader::convertToT( const std::string& element, - std::string& val) -{ - val = element; - return true; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Conversion function for command line arguments of type bool -//////////////////////////////////////////////////////////////////////////////// -template<> -inline bool -CmdArgReader::convertToT( const std::string& element, bool& val) -{ - // check if value is given as string-type { true | false } - if ( "true" == element) - { - val = true; - return true; - } - else if ( "false" == element) - { - val = false; - return true; - } - // check if argument is given as integer { 0 | 1 } - else - { - int tmp; - if ( convertToT( element, tmp)) - { - if ( 1 == tmp) - { - val = true; - return true; - } - else if ( 0 == tmp) - { - val = false; - return true; - } - } - } - - return false; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Get the value of the command line argument with given name -//! @return A const handle to the requested argument. If the argument does -//! not exist or if it is not from type T NULL is returned -//! @param T the type of the argument requested -//! @param name the name of the requested argument -//////////////////////////////////////////////////////////////////////////////// -template -/*static*/ const T* -CmdArgReader::getArg( const std::string& name) -{ - if( ! self) - { - RUNTIME_EXCEPTION("CmdArgReader::getArg(): CmdArgReader not initialized."); - return NULL; - } - - return self->getArgHelper( name); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Check if a command line argument with the given name exists -//! @return true if a command line argument with name \a name exists, -//! otherwise false -//! @param name name of the command line argument in question -//////////////////////////////////////////////////////////////////////////////// -/*static*/ inline bool -CmdArgReader::existArg( const std::string& name) -{ - if( ! self) - { - RUNTIME_EXCEPTION("CmdArgReader::getArg(): CmdArgReader not initialized."); - return false; - } - - return self->existArgHelper( name); -} - -//////////////////////////////////////////////////////////////////////////////// -//! @brief Get the value of the command line argument with given name -//! @return A const handle to the requested argument. If the argument does -//! not exist or if it is not from type T NULL is returned -//! @param T the type of the argument requested -//! @param name the name of the requested argument -//////////////////////////////////////////////////////////////////////////////// -template -const T* -CmdArgReader::getArgHelper( const std::string& name) -{ - // check if argument already processed and stored in correct type - if ( args.end() != (iter = args.find( name))) - { - if ( (*(iter->second.first)) == typeid( T) ) - { - return (T*) iter->second.second; - } - } - else - { - T* tmp = new T; - - // check the array with unprocessed values - if ( unprocessed.end() != (iter_unprocessed = unprocessed.find( name))) - { - // try to "cast" the string to the type requested - if ( convertToT< T >( iter_unprocessed->second, *tmp)) - { - // add the token element pair to map of already converted values - args[name] = std::make_pair( &(typeid( T)), (void*) tmp); - - return tmp; - } - } - - // not used while not inserted into the map -> cleanup - delete tmp; - } - - // failed, argument not available - return NULL; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Check if a command line argument with name \a name exists -//! @return true if a command line argument of name \a name exists, -//! otherwise false -//! @param name the name of the requested argument -//////////////////////////////////////////////////////////////////////////////// -inline bool -CmdArgReader::existArgHelper( const std::string& name) const -{ - bool ret_val = false; - - // check if argument already processed and stored in correct type - if( args.end() != args.find( name)) - { - ret_val = true; - } - else - { - - // check the array with unprocessed values - if ( unprocessed.end() != unprocessed.find( name)) - { - ret_val = true; - } - } - - return ret_val; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Get the original / raw argc program argument -//////////////////////////////////////////////////////////////////////////////// -/*static*/ inline int& -CmdArgReader::getRArgc() -{ - if( ! self) - { - RUNTIME_EXCEPTION("CmdArgReader::getRArgc(): CmdArgReader not initialized."); - } - - return rargc; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Get the original / raw argv program argument -//////////////////////////////////////////////////////////////////////////////// -/*static*/ inline char**& -CmdArgReader::getRArgv() -{ - if( ! self) - { - RUNTIME_EXCEPTION("CmdArgReader::getRArgc(): CmdArgReader not initialized."); - } - - return rargv; -} - -// functions, exported (extern) - -#endif // #ifndef _CMDARGREADER_H_ diff --git a/tests/opencl/vectorhypot/exception.h b/tests/opencl/vectorhypot/exception.h deleted file mode 100644 index e4650d99..00000000 --- a/tests/opencl/vectorhypot/exception.h +++ /dev/null @@ -1,151 +0,0 @@ -/* -* Copyright 1993-2010 NVIDIA Corporation. All rights reserved. -* -* Please refer to the NVIDIA end user license agreement (EULA) associated -* with this source code for terms and conditions that govern your use of -* this software. Any use, reproduction, disclosure, or distribution of -* this software and related documentation outside the terms of the EULA -* is strictly prohibited. -* -*/ - -/* CUda UTility Library */ -#ifndef _EXCEPTION_H_ -#define _EXCEPTION_H_ - -// includes, system -#include -#include -#include -#include - -//! Exception wrapper. -//! @param Std_Exception Exception out of namespace std for easy typing. -template -class Exception : public Std_Exception -{ -public: - - //! @brief Static construction interface - //! @return Alwayss throws ( Located_Exception) - //! @param file file in which the Exception occurs - //! @param line line in which the Exception occurs - //! @param detailed details on the code fragment causing the Exception - static void throw_it( const char* file, - const int line, - const char* detailed = "-" ); - - //! Static construction interface - //! @return Alwayss throws ( Located_Exception) - //! @param file file in which the Exception occurs - //! @param line line in which the Exception occurs - //! @param detailed details on the code fragment causing the Exception - static void throw_it( const char* file, - const int line, - const std::string& detailed); - - //! Destructor - virtual ~Exception() throw(); - -private: - - //! Constructor, default (private) - Exception(); - - //! Constructor, standard - //! @param str string returned by what() - Exception( const std::string& str); - -}; - -//////////////////////////////////////////////////////////////////////////////// -//! Exception handler function for arbitrary exceptions -//! @param ex exception to handle -//////////////////////////////////////////////////////////////////////////////// -template -inline void -handleException( const Exception_Typ& ex) -{ - std::cerr << ex.what() << std::endl; - - exit( EXIT_FAILURE); -} - -//! Convenience macros - -//! Exception caused by dynamic program behavior, e.g. file does not exist -#define RUNTIME_EXCEPTION( msg) \ - Exception::throw_it( __FILE__, __LINE__, msg) - -//! Logic exception in program, e.g. an assert failed -#define LOGIC_EXCEPTION( msg) \ - Exception::throw_it( __FILE__, __LINE__, msg) - -//! Out of range exception -#define RANGE_EXCEPTION( msg) \ - Exception::throw_it( __FILE__, __LINE__, msg) - -//////////////////////////////////////////////////////////////////////////////// -//! Implementation - -// includes, system -#include - -//////////////////////////////////////////////////////////////////////////////// -//! Static construction interface. -//! @param Exception causing code fragment (file and line) and detailed infos. -//////////////////////////////////////////////////////////////////////////////// -/*static*/ template -void -Exception:: -throw_it( const char* file, const int line, const char* detailed) -{ - std::stringstream s; - - // Quiet heavy-weight but exceptions are not for - // performance / release versions - s << "Exception in file '" << file << "' in line " << line << "\n" - << "Detailed description: " << detailed << "\n"; - - throw Exception( s.str()); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Static construction interface. -//! @param Exception causing code fragment (file and line) and detailed infos. -//////////////////////////////////////////////////////////////////////////////// -/*static*/ template -void -Exception:: -throw_it( const char* file, const int line, const std::string& msg) -{ - throw_it( file, line, msg.c_str()); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Constructor, default (private). -//////////////////////////////////////////////////////////////////////////////// -template -Exception::Exception() : - Exception("Unknown Exception.\n") -{ } - -//////////////////////////////////////////////////////////////////////////////// -//! Constructor, standard (private). -//! String returned by what(). -//////////////////////////////////////////////////////////////////////////////// -template -Exception::Exception( const std::string& s) : - Std_Exception( s) -{ } - -//////////////////////////////////////////////////////////////////////////////// -//! Destructor -//////////////////////////////////////////////////////////////////////////////// -template -Exception::~Exception() throw() { } - -// functions, exported - -#endif // #ifndef _EXCEPTION_H_ - diff --git a/tests/opencl/vectorhypot/kernel.cl b/tests/opencl/vectorhypot/kernel.cl deleted file mode 100644 index 1983a862..00000000 --- a/tests/opencl/vectorhypot/kernel.cl +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 1993-2010 NVIDIA Corporation. All rights reserved. - * - * Please refer to the NVIDIA end user license agreement (EULA) associated - * with this source code for terms and conditions that govern your use of - * this software. Any use, reproduction, disclosure, or distribution of - * this software and related documentation outside the terms of the EULA - * is strictly prohibited. - * - */ - -// OpenCL Kernel Function Naive Implementation for hyptenuse -__kernel void VectorHypot(__global float4* fg4A, __global float4* fg4B, __global float4* fg4Hypot, unsigned int uiOffset, int iInnerLoopCount, unsigned int uiNumElements) -{ - // get index into global data array - size_t szGlobalOffset = get_global_id(0) + uiOffset; - - // bound check - if (szGlobalOffset >= uiNumElements) - { - return; - } - - // Processing 4 elements per work item, so read fgA and fgB source values from GMEM - float4 f4A = fg4A[szGlobalOffset]; - float4 f4B = fg4B[szGlobalOffset]; - float4 f4H = (float4)0.0f; - - // Get the hypotenuses the vectors of 'legs', but exaggerate the time needed with loop - for (int i = 0; i < iInnerLoopCount; i++) - { - // compute the 4 hypotenuses using built-in function - f4H.x = hypot (f4A.x, f4B.x); - f4H.y = hypot (f4A.y, f4B.y); - f4H.z = hypot (f4A.z, f4B.z); - f4H.w = hypot (f4A.w, f4B.w); - } - - // Write 4 result values back out to GMEM - fg4Hypot[szGlobalOffset] = f4H; -} \ No newline at end of file diff --git a/tests/opencl/vectorhypot/main.cc b/tests/opencl/vectorhypot/main.cc deleted file mode 100644 index ec25bbf2..00000000 --- a/tests/opencl/vectorhypot/main.cc +++ /dev/null @@ -1,702 +0,0 @@ -/* - * Copyright 1993-2010 NVIDIA Corporation. All rights reserved. - * - * Please refer to the NVIDIA end user license agreement (EULA) associated - * with this source code for terms and conditions that govern your use of - * this software. Any use, reproduction, disclosure, or distribution of - * this software and related documentation outside the terms of the EULA - * is strictly prohibited. - * - */ - -// ********************************************************************* -// oclCopyComputeOverlap Notes: -// -// OpenCL API demo application for NVIDIA CUDA GPU's that implements a -// element by element vector hyptenuse computation using 2 input float arrays -// and 1 output float array. -// -// Demonstrates host->GPU and GPU->host copies that are asynchronous/overlapped -// with respect to GPU computation (and with respect to host thread). -// -// Because the overlap acheivable for this computation and data set on a given system depends upon the GPU being used and the -// GPU/Host bandwidth, the sample adjust the computation duration to test the most ideal case and test against a consistent standard. -// This sample should be able to achieve up to 30% overlap on GPU's arch 1.2 and 1.3, and up to 50% on arch 2.0+ (Fermi) GPU's. -// -// After setup, warmup and calibration to the system, the sample runs 4 scenarios: -// A) Computations with 2 command queues on GPU -// A multiple-cycle sequence is executed, timed and compared against the host -// B) Computations with 1 command queue on GPU -// A multiple-cycle sequence is executed, timed and compared against the host -// -// The 2-command queue approach ought to be substantially faster -// -// For developmental purposes, the "iInnerLoopCount" variable passes into kernel and independently -// increases compute time without increasing data size (via a loop inside the kernel) -// -// At some value of iInnerLoopCount, # of elements, workgroup size, etc the Overlap percentage should reach 30%: -// (This ~naively assumes time H2D bandwidth is the same as D2H bandwidth, but this is close on most systems) -// -// If we name the time to copy single input vector H2D (or outpute vector D2H) as "T", then the optimum comparison case is: -// -// Single Queue with all the data and all the work -// Ttot (serial) = 4T + 4T + 2T = 10T -// -// Dual Queue, where each queue has 1/2 the data and 1/2 the work -// Tq0 (overlap) = 2T + 2T + T .... -// Tq1 (overlap) = .... 2T + 2T + T -// -// Ttot (elapsed, wall) = 2T + 2T + 2T + T = 7T -// -// Best Overlap % = 100.0 * (10T - 7T)/10T = 30.0 % (Tesla arch 1.2 or 1.3, single copy engine) -// -// For multiple independent cycles using arch >= 2.0 with 2 copy engines, input and output copies can also be overlapped. -// This doesn't help for the first cycle, but theoretically can lead to 50% overlap over many independent cycles. -// ********************************************************************* - -// common SDK header for standard utilities and system libs -#include -#include -#include - -// Best possible and Min ratio of compute/copy overlap timing benefit to pass the test -// values greater than 0.0f represent a speed-up relative to non-overlapped -#define EXPECTED_OVERLAP 30.0f -#define EXPECTED_OVERLAP_FERMI 45.0f -#define PASS_FACTOR 0.60f -#define RETRIES_ON_FAILURE 1 - -// Base sizes for parameters manipulated dynamically or on the command line -#define BASE_WORK_ITEMS 64 -#define BASE_ARRAY_LENGTH 40000 -#define BASE_LOOP_COUNT 32 - -static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) { - if (nullptr == filename || nullptr == data || 0 == size) - return CL_INVALID_VALUE; - - FILE* fp = fopen(filename, "r"); - if (NULL == fp) { - fprintf(stderr, "Failed to load kernel."); - return CL_INVALID_VALUE; - } - fseek(fp , 0 , SEEK_END); - long fsize = ftell(fp); - rewind(fp); - - *data = (uint8_t*)malloc(fsize); - *size = fread(*data, 1, fsize, fp); - - fclose(fp); - - return CL_SUCCESS; -} - -// Vars -// ********************************************************************* -cl_platform_id cpPlatform; // OpenCL platform -cl_context cxGPUContext; // OpenCL context -cl_command_queue cqCommandQueue[2]; // OpenCL command queues -cl_device_id* cdDevices; // OpenCL device list -cl_program cpProgram; // OpenCL program -cl_kernel ckKernel[2]; // OpenCL kernel, 1 per queue -cl_mem cmPinnedSrcA; // OpenCL pinned host source buffer A -cl_mem cmPinnedSrcB; // OpenCL pinned host source buffer B -cl_mem cmPinnedResult; // OpenCL pinned host result buffer -float* fSourceA = NULL; // Mapped pointer for pinned Host source A buffer -float* fSourceB = NULL; // Mapped pointer for pinned Host source B buffer -float* fResult = NULL; // Mapped pointer for pinned Host result buffer -cl_mem cmDevSrcA; // OpenCL device source buffer A -cl_mem cmDevSrcB; // OpenCL device source buffer B -cl_mem cmDevResult; // OpenCL device result buffer -size_t szBuffBytes; // Size of main buffers -size_t szGlobalWorkSize; // 1D var for Total # of work items in the launched ND range -size_t szLocalWorkSize = BASE_WORK_ITEMS; // initial # of work items in the work group -cl_int ciErrNum; // Error code var -char* cPathAndName = NULL; // Var for full paths to data, src, etc. -char* cSourceCL = NULL; // Buffer to hold source for compilation -const char* cExecutableName = NULL; - -// demo config vars -const char* cSourceFile = "kernel.cl"; // OpenCL computation kernel source code -float* Golden = NULL; // temp buffer to hold golden results for cross check -bool bNoPrompt = false; // Command line switch to skip exit prompt -bool bQATest = false; // Command line switch to test - -// Forward Declarations -// ********************************************************************* -double DualQueueSequence(int iCycles, unsigned int uiNumElements, bool bShowConfig); -double OneQueueSequence(int iCycles, unsigned int uiNumElements, bool bShowConfig); -int AdjustCompute(cl_device_id cdTargetDevice, unsigned int uiNumElements, int iInitialLoopCount, int iCycles); -void VectorHypotHost(const float* pfData1, const float* pfData2, float* pfResult, unsigned int uiNumElements, int iInnerLoopCount); -void Cleanup (int iExitCode); -void (*pCleanup)(int) = &Cleanup; - -int *gp_argc = 0; -const char *** gp_argv = NULL; - -// Main function -// ********************************************************************* -int main(int argc, const char **argv) -{ - //Locals - size_t szKernelLength; // Byte size of kernel code - double dBuildTime; // Compile time - cl_uint uiTargetDevice = 0; // Default Device to compute on - cl_uint uiNumDevsUsed = 1; // Number of devices used in this sample - cl_uint uiNumDevices; // Number of devices available - int iDevCap = -1; // Capability of device - int iInnerLoopCount = BASE_LOOP_COUNT; // Varies "compute intensity" per data within the kernel - const int iTestCycles = 10; // How many times to run the external test loop - const int iWarmupCycles = 8; // How many times to run the warmup sequence - cl_uint uiWorkGroupMultiple = 4; // Command line var (using "workgroupmult=") to optionally increase workgroup size - cl_uint uiNumElements = BASE_ARRAY_LENGTH; // initial # of elements per array to process (note: procesing 4 per work item) - cl_uint uiSizeMultiple = 4; // Command line var (using "sizemult=") to optionally increase vector sizes - bool bPassFlag = false; // Var to accumulate test pass/fail - shrBOOL bMatch = shrFALSE; // Cross check result - shrBOOL bTestOverlap = shrFALSE; - double dAvgGPUTime[2] = {0.0, 0.0}; // Average time of iTestCycles calls for 2-Queue and 1-Queue test - double dHostTime[2] = {0.0, 0.0}; // Host computation time (2nd test is redundant but a good stability indicator) - float fMinPassCriteria[2] = {0.0f, 0.0f}; // Test pass cireria, adjusted dependant on GPU arch - - gp_argc = &argc; - gp_argv = &argv; - - shrQAStart(argc, (char **)argv); - - // start logs - cExecutableName = argv[0]; - shrSetLogFileName ("oclCopyComputeOverlap.txt"); - shrLog("%s Starting...\n\n", argv[0]); - - // get basic command line args - bNoPrompt = (shrTRUE == shrCheckCmdLineFlag(argc, argv, "noprompt")); - bQATest = (shrTRUE == shrCheckCmdLineFlag(argc, argv, "qatest")); - shrGetCmdLineArgumentu(argc, argv, "device", &uiTargetDevice); - - // Optional Command-line multiplier for vector size - // Default val of 4 gives 10.24 million float elements per vector - // Range of 3 - 16 (7.68 to 40.96 million floats) is reasonable range (if system and GPU have enough memory) - shrGetCmdLineArgumentu(argc, argv, "sizemult", &uiSizeMultiple); - uiSizeMultiple = CLAMP(uiSizeMultiple, 1, 50); - uiNumElements = uiSizeMultiple * BASE_ARRAY_LENGTH * BASE_WORK_ITEMS; - shrLog("Array sizes = %u float elements\n", uiNumElements); - - // Optional Command-line multiplier for workgroup size (x 64 work items) - // Default val of 4 gives szLocalWorkSize of 256. - // Range of 1 - 8 (resulting in workgroup sizes of 64 to 512) is reasonable range - shrGetCmdLineArgumentu(argc, argv, "workgroupmult", &uiWorkGroupMultiple); - uiWorkGroupMultiple = CLAMP(uiWorkGroupMultiple, 1, 10); - szLocalWorkSize = uiWorkGroupMultiple * BASE_WORK_ITEMS; - shrLog("Workgroup Size = %u\n\n", szLocalWorkSize); - - // Get the NVIDIA platform if available, otherwise use default - shrLog("Get the Platform ID...\n\n"); - ciErrNum = oclGetPlatformID(&cpPlatform); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - - // Get OpenCL platform name and version - char cBuffer[256]; - ciErrNum = clGetPlatformInfo (cpPlatform, CL_PLATFORM_NAME, sizeof(cBuffer), cBuffer, NULL); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - shrLog("Platform Name = %s\n\n", cBuffer); - - // Get all the devices - shrLog("Get the Device info and select Device...\n"); - uiNumDevices = 1; - cdDevices = (cl_device_id*)malloc(uiNumDevices * sizeof(cl_device_id)); - ciErrNum = clGetDeviceIDs(cpPlatform, CL_DEVICE_TYPE_DEFAULT, 1, cdDevices, NULL); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - - // Set target device and check capabilities - shrLog(" # of Devices Available = %u\n", uiNumDevices); - uiTargetDevice = CLAMP(uiTargetDevice, 0, (uiNumDevices - 1)); - shrLog(" Using Device %u, ", uiTargetDevice); - oclPrintDevName(LOGBOTH, cdDevices[uiTargetDevice]); - /*iDevCap = oclGetDevCap(cdDevices[uiTargetDevice]); - if (iDevCap > 0) { - shrLog(", Capability = %d.%d\n\n", iDevCap/10, iDevCap%10); - } else { - shrLog("\n\n", iDevCap); - } - if (strstr(cBuffer, "NVIDIA") != NULL) - { - if (iDevCap < 12) - { - shrLog("Device doesn't have overlap capability. Skipping test...\n"); - Cleanup (EXIT_SUCCESS); - } - - // Device and Platform eligible for overlap testing - bTestOverlap = shrTRUE; - - // If device has overlap capability, proceed - fMinPassCriteria[0] = PASS_FACTOR * EXPECTED_OVERLAP; // 1st cycle overlap is same for 1 or 2 copy engines - if (iDevCap != 20) - { - // Single copy engine - fMinPassCriteria[1] = PASS_FACTOR * EXPECTED_OVERLAP; // avg of many cycles - } - else - { - char cDevName[1024]; - clGetDeviceInfo(cdDevices[uiTargetDevice], CL_DEVICE_NAME, sizeof(cDevName), &cDevName, NULL); - if(strstr(cDevName, "Quadro")!=0 || strstr(cDevName, "Tesla")!=0) - { - // Tesla or Quadro (arch = 2.0) ... Dual copy engine - fMinPassCriteria[1] = PASS_FACTOR * EXPECTED_OVERLAP_FERMI; // average of many cycles - } - else - { - // Geforce ... Single copy engine - fMinPassCriteria[1] = PASS_FACTOR * EXPECTED_OVERLAP; // average of many cycles - } - } - }*/ - - // Create the context - shrLog("clCreateContext...\n"); - cxGPUContext = clCreateContext(0, uiNumDevsUsed, &cdDevices[uiTargetDevice], NULL, NULL, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - - // Create 2 command-queues - cqCommandQueue[0] = clCreateCommandQueue(cxGPUContext, cdDevices[uiTargetDevice], 0, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - shrLog("clCreateCommandQueue [0]...\n"); - cqCommandQueue[1] = clCreateCommandQueue(cxGPUContext, cdDevices[uiTargetDevice], 0, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - shrLog("clCreateCommandQueue [1]...\n"); - - // Allocate the OpenCL source and result buffer memory objects on GPU device GMEM - szBuffBytes = sizeof(cl_float) * uiNumElements; - cmDevSrcA = clCreateBuffer(cxGPUContext, CL_MEM_READ_ONLY, szBuffBytes, NULL, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - cmDevSrcB = clCreateBuffer(cxGPUContext, CL_MEM_READ_ONLY, szBuffBytes, NULL, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - cmDevResult = clCreateBuffer(cxGPUContext, CL_MEM_WRITE_ONLY, szBuffBytes, NULL, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - shrLog("clCreateBuffer (Src A, Src B and Result GPU Device GMEM, 3 x %u floats) ...\n", uiNumElements); - - // Allocate pinned source and result host buffers: - // Note: Pinned (Page Locked) memory is needed for async host<->GPU memory copy operations *** - cmPinnedSrcA = clCreateBuffer(cxGPUContext, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, szBuffBytes, NULL, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - cmPinnedSrcB = clCreateBuffer(cxGPUContext, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, szBuffBytes, NULL, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - cmPinnedResult = clCreateBuffer(cxGPUContext, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, szBuffBytes, NULL, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - shrLog("clCreateBuffer (Src A, Src B and Result Pinned Host buffers, 3 x %u floats)...\n\n", uiNumElements); - - // Get mapped pointers to pinned input host buffers - // Note: This allows general (non-OpenCL) host functions to access pinned buffers using standard pointers - fSourceA = (cl_float*)clEnqueueMapBuffer(cqCommandQueue[0], cmPinnedSrcA, CL_TRUE, CL_MAP_WRITE, 0, szBuffBytes, 0, NULL, NULL, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - fSourceB = (cl_float*)clEnqueueMapBuffer(cqCommandQueue[0], cmPinnedSrcB, CL_TRUE, CL_MAP_WRITE, 0, szBuffBytes, 0, NULL, NULL, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - fResult = (cl_float*)clEnqueueMapBuffer(cqCommandQueue[0], cmPinnedResult, CL_TRUE, CL_MAP_READ, 0, szBuffBytes, 0, NULL, NULL, &ciErrNum); - oclCheckErrorEX (ciErrNum, CL_SUCCESS, pCleanup); - shrLog("clEnqueueMapBuffer (Pointers to 3 pinned host buffers)...\n"); - - // Alloc temp golden buffer for cross checks - Golden = (float*)malloc(szBuffBytes); - oclCheckErrorEX(Golden != NULL, shrTRUE, pCleanup); - -#ifdef HOSTGPU - // Read the OpenCL kernel in from source file - cPathAndName = shrFindFilePath(cSourceFile, argv[0]); - oclCheckError(cPathAndName != NULL, shrTRUE); - shrLog("oclLoadProgSource (%s)...\n", cSourceFile); - cSourceCL = oclLoadProgSource(cPathAndName, "", &szKernelLength); - // Create the program object - shrLog("clCreateProgramWithSource...\n"); - cpProgram = clCreateProgramWithSource(cxGPUContext, 1, (const char **)&cSourceCL, &szKernelLength, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); -#else - uint8_t *kernel_bin = NULL; - size_t kernel_size; - cl_int binary_status = 0; - ciErrNum = read_kernel_file("kernel.pocl", &kernel_bin, &kernel_size); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - cpProgram = clCreateProgramWithBinary( - cxGPUContext, 1, &cdDevices[uiTargetDevice], &kernel_size, (const uint8_t**)&kernel_bin, &binary_status, &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); -#endif - // Build the program for the target device - clFinish(cqCommandQueue[0]); - shrDeltaT(0); - ciErrNum = clBuildProgram(cpProgram, uiNumDevsUsed, &cdDevices[uiTargetDevice], "-cl-fast-relaxed-math", NULL, NULL); - shrLog("clBuildProgram..."); - if (ciErrNum != CL_SUCCESS) - { - // write out standard error, Build Log and PTX, then cleanup and exit - shrLogEx(LOGBOTH | ERRORMSG, (double)ciErrNum, STDERROR); - oclLogBuildInfo(cpProgram, oclGetFirstDev(cxGPUContext)); - oclLogPtx(cpProgram, oclGetFirstDev(cxGPUContext), "VectorHypot.ptx"); - Cleanup(EXIT_FAILURE); - } - dBuildTime = shrDeltaT(0); - - // Create the kernel - ckKernel[0] = clCreateKernel(cpProgram, "VectorHypot", &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - ckKernel[1] = clCreateKernel(cpProgram, "VectorHypot", &ciErrNum); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - shrLog("clCreateKernel (ckKernel[2])...\n"); - - // Offsets for 2 queues - cl_uint uiOffset[2] = {0, uiNumElements / (2 * 4)}; - - // Set the Argument values for the 1st kernel instance (queue 0) - ciErrNum = clSetKernelArg(ckKernel[0], 0, sizeof(cl_mem), (void*)&cmDevSrcA); - ciErrNum |= clSetKernelArg(ckKernel[0], 1, sizeof(cl_mem), (void*)&cmDevSrcB); - ciErrNum |= clSetKernelArg(ckKernel[0], 2, sizeof(cl_mem), (void*)&cmDevResult); - ciErrNum |= clSetKernelArg(ckKernel[0], 3, sizeof(cl_uint), (void*)&uiOffset[0]); - ciErrNum |= clSetKernelArg(ckKernel[0], 4, sizeof(cl_int), (void*)&iInnerLoopCount); - ciErrNum |= clSetKernelArg(ckKernel[0], 5, sizeof(cl_uint), (void*)&uiNumElements); - shrLog("clSetKernelArg ckKernel[0] args 0 - 5...\n"); - - // Set the Argument values for the 2d kernel instance (queue 1) - ciErrNum |= clSetKernelArg(ckKernel[1], 0, sizeof(cl_mem), (void*)&cmDevSrcA); - ciErrNum |= clSetKernelArg(ckKernel[1], 1, sizeof(cl_mem), (void*)&cmDevSrcB); - ciErrNum |= clSetKernelArg(ckKernel[1], 2, sizeof(cl_mem), (void*)&cmDevResult); - ciErrNum |= clSetKernelArg(ckKernel[1], 3, sizeof(cl_uint), (void*)&uiOffset[1]); - ciErrNum |= clSetKernelArg(ckKernel[1], 4, sizeof(cl_int), (void*)&iInnerLoopCount); - ciErrNum |= clSetKernelArg(ckKernel[1], 5, sizeof(cl_uint), (void*)&uiNumElements); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - shrLog("clSetKernelArg ckKernel[1] args 0 - 5...\n\n"); - - //******************************************* - // Warmup the driver with dual queue sequence - //******************************************* - - // Warmup with dual queue sequence for iTestCycles - shrLog("Warmup with 2-Queue sequence, %d cycles...\n", iWarmupCycles); - DualQueueSequence(iWarmupCycles, uiNumElements, false); - - // Use single queue config to adjust compute intensity - shrLog("Adjust compute for GPU / system...\n"); - iInnerLoopCount = AdjustCompute(cdDevices[uiTargetDevice], uiNumElements, iInnerLoopCount, iTestCycles); - shrLog(" Kernel inner loop count = %d\n", iInnerLoopCount); - - //******************************************* - // Run and time with 2 command-queues - //******************************************* - for( int iRun =0; iRun <= RETRIES_ON_FAILURE; ++iRun ) { - - // Run the sequence iTestCycles times - dAvgGPUTime[0] = DualQueueSequence(iTestCycles, uiNumElements, false); - - // Warmup then Compute on host iTestCycles times (using mapped standard pointer to pinned host cl_mem buffer) - shrLog(" Device vs Host Result Comparison\t: "); - VectorHypotHost(fSourceA, fSourceB, Golden, uiNumElements, iInnerLoopCount); - shrDeltaT(0); - for (int i = 0; i < iTestCycles; i++) - { - VectorHypotHost (fSourceA, fSourceB, Golden, uiNumElements, iInnerLoopCount); - } - dHostTime[0] = shrDeltaT(0)/iTestCycles; - - // Compare host and GPU results (using mapped standard pointer to pinned host cl_mem buffer) - bMatch = shrComparefet(Golden, fResult, uiNumElements, 0.0f, 0); - shrLog("gpu %s cpu\n", (bMatch == shrTRUE) ? "MATCHES" : "DOESN'T MATCH"); - bPassFlag = (bMatch == shrTRUE); - - //******************************************* - // Run and time with 1 command queue - //******************************************* - // Run the sequence iTestCycles times - dAvgGPUTime[1] = OneQueueSequence(iTestCycles, uiNumElements, false); - - // Compute on host iTestCycles times (using mapped standard pointer to pinned host cl_mem buffer) - shrLog(" Device vs Host Result Comparison\t: "); - shrDeltaT(0); - for (int i = 0; i < iTestCycles; i++) - { - VectorHypotHost(fSourceA, fSourceB, Golden, (int)uiNumElements, iInnerLoopCount); - } - dHostTime[1] = shrDeltaT(0)/iTestCycles; - - // Compare host and GPU results (using mapped standard pointer to pinned host cl_mem buffer) - bMatch = shrComparefet(Golden, fResult, uiNumElements, 0.0f, 0); - shrLog("gpu %s cpu\n", (bMatch == shrTRUE) ? "MATCHES" : "DOESN'T MATCH"); - bPassFlag &= (bMatch == shrTRUE); - - //******************************************* - - // Compare Single and Dual queue timing - shrLog("\nResult Summary:\n"); - - // Log GPU and CPU Time for 2-queue scenario - shrLog(" Avg GPU Elapsed Time for 2-Queues\t= %.5f s\n", dAvgGPUTime[0]); - shrLog(" Avg Host Elapsed Time\t\t\t= %.5f s\n\n", dHostTime[0]); - - // Log GPU and CPU Time for 1-queue scenario - shrLog(" Avg GPU Elapsed Time for 1-Queue\t= %.5f s\n", dAvgGPUTime[1]); - shrLog(" Avg Host Elapsed Time\t\t\t= %.5f s\n\n", dHostTime[1]); - - // Log overlap % for GPU (comparison of 2-queue and 1 queue scenarios) and status - double dAvgOverlap = 100.0 * (1.0 - dAvgGPUTime[0]/dAvgGPUTime[1]); - - if( bTestOverlap ) { - bool bAvgOverlapOK = (dAvgOverlap >= fMinPassCriteria[1]); - if( iRun == RETRIES_ON_FAILURE || bAvgOverlapOK ) { - shrLog(" Measured and (Acceptable) Avg Overlap\t= %.1f %% (%.1f %%) -> Measured Overlap is %s\n\n", dAvgOverlap, fMinPassCriteria[1], bAvgOverlapOK ? "Acceptable" : "NOT Acceptable"); - - // Log info to master log in standard format - shrLogEx(LOGBOTH | MASTER, 0, "oclCopyComputeOverlap-Avg, Throughput = %.4f OverlapPercent, Time = %.5f s, Size = %u Elements, NumDevsUsed = %u, Workgroup = %u\n", - dAvgOverlap, dAvgGPUTime[0], uiNumElements, uiNumDevsUsed, szLocalWorkSize); - - bPassFlag &= bAvgOverlapOK; - break; - } - } - - shrLog(" Measured and (Acceptable) Avg Overlap\t= %.1f %% (%.1f %%) -> Retry %d more time(s)...\n\n", dAvgOverlap, fMinPassCriteria[1], RETRIES_ON_FAILURE - iRun); - } - - - //******************************************* - // Report pass/fail, cleanup and exit - Cleanup (bPassFlag ? EXIT_SUCCESS : EXIT_FAILURE); - - return 0; -} - -// Run 1 queue sequence for n cycles -// ********************************************************************* -double OneQueueSequence(int iCycles, unsigned int uiNumElements, bool bShowConfig) -{ - // Use fresh source Data: (re)initialize pinned host array buffers (using mapped standard pointer to pinned host cl_mem buffer) - shrFillArray(fSourceA, (int)uiNumElements); - shrFillArray(fSourceB, (int)uiNumElements); - - // Reset Global work size for 1 command-queue, and log work sizes & dimensions - szGlobalWorkSize = shrRoundUp((int)szLocalWorkSize, (int)(uiNumElements/4)); - - // *** Make sure queues are empty and then start timer - double dAvgTime = 0.0; - clFinish(cqCommandQueue[0]); - clFinish(cqCommandQueue[1]); - shrDeltaT(0); - - // Run the sequence iCycles times - for (int i = 0; i < iCycles; i++) - { - // Nonblocking Write of all of input data from host to device in command-queue 0 - ciErrNum = clEnqueueWriteBuffer(cqCommandQueue[0], cmDevSrcA, CL_FALSE, 0, szBuffBytes, (void*)&fSourceA[0], 0, NULL, NULL); - ciErrNum |= clEnqueueWriteBuffer(cqCommandQueue[0], cmDevSrcB, CL_FALSE, 0, szBuffBytes, (void*)&fSourceB[0], 0, NULL, NULL); - shrCheckError(ciErrNum, CL_SUCCESS); - - // Launch kernel computation, command-queue 0 - ciErrNum = clEnqueueNDRangeKernel(cqCommandQueue[0], ckKernel[0], 1, NULL, &szGlobalWorkSize, &szLocalWorkSize, 0, NULL, NULL); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - - // Non Blocking Read of output data from device to host, command-queue 0 - ciErrNum = clEnqueueReadBuffer(cqCommandQueue[0], cmDevResult, CL_FALSE, 0, szBuffBytes, (void*)&fResult[0], 0, NULL, NULL); - shrCheckError(ciErrNum, CL_SUCCESS); - - // Flush sequence to device (may not be necessary on Linux or WinXP or when using the NVIDIA Tesla Computing Cluster driver) - clFlush(cqCommandQueue[0]); - } - - // *** Assure sync to host and return average sequence time - clFinish(cqCommandQueue[0]); - dAvgTime = shrDeltaT(0)/(double)iCycles; - - // Log config if asked for - if (bShowConfig) - { - shrLog("\n1-Queue sequence Configuration:\n"); - shrLog(" Global Work Size (per command-queue)\t= %u\n Local Work Size \t\t\t= %u\n # of Work Groups (per command-queue)\t= %u\n # of command-queues\t\t\t= 1\n", - szGlobalWorkSize, szLocalWorkSize, szGlobalWorkSize/szLocalWorkSize); - } - return dAvgTime; -} - -// Run 2 queue sequence for n cycles -// ********************************************************************* -double DualQueueSequence(int iCycles, unsigned int uiNumElements, bool bShowConfig) -{ - // Locals - size_t szHalfBuffer = szBuffBytes / 2; - size_t szHalfOffset = szHalfBuffer / sizeof(float); - double dAvgTime = 0.0; - - // Use fresh source Data: (re)initialize pinned host array buffers (using mapped standard pointer to pinned host cl_mem buffer) - shrFillArray(fSourceA, (int)uiNumElements); - shrFillArray(fSourceB, (int)uiNumElements); - - // Set Global work size for 2 command-queues, and log work sizes & dimensions - szGlobalWorkSize = shrRoundUp((int)szLocalWorkSize, (int)(uiNumElements/(2 * 4))); - - // Make sure queues are empty and then start timer - clFinish(cqCommandQueue[0]); - clFinish(cqCommandQueue[1]); - shrDeltaT(0); - - for (int i = 0; i < iCycles; i++) - { - // Mid Phase 0 - // Nonblocking Write of 1st half of input data from host to device in command-queue 0 - ciErrNum = clEnqueueWriteBuffer(cqCommandQueue[0], cmDevSrcA, CL_FALSE, 0, szHalfBuffer, (void*)&fSourceA[0], 0, NULL, NULL); - ciErrNum |= clEnqueueWriteBuffer(cqCommandQueue[0], cmDevSrcB, CL_FALSE, 0, szHalfBuffer, (void*)&fSourceB[0], 0, NULL, NULL); - shrCheckError(ciErrNum, CL_SUCCESS); - - // Push out the write for queue 0 (and prior read from queue 1 at end of loop) to the driver - // (not necessary on Linux, Mac OSX or WinXP) - clFlush(cqCommandQueue[0]); - clFlush(cqCommandQueue[1]); - - // Start Phase 1 *********************************** - - // Launch kernel computation, command-queue 0 - // (Note: The order MATTERS here on Fermi ! THE KERNEL IN THIS PHASE SHOULD BE LAUNCHED BEFORE THE WRITE) - ciErrNum = clEnqueueNDRangeKernel(cqCommandQueue[0], ckKernel[0], 1, NULL, &szGlobalWorkSize, &szLocalWorkSize, 0, NULL, NULL); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - - // Nonblocking Write of 2nd half of input data from host to device in command-queue 1 - // (Note: The order MATTERS here on Fermi ! THE KERNEL IN THIS PHASE SHOULD BE LAUNCHED BEFORE THE WRITE) - ciErrNum = clEnqueueWriteBuffer(cqCommandQueue[1], cmDevSrcA, CL_FALSE, szHalfBuffer, szHalfBuffer, (void*)&fSourceA[szHalfOffset], 0, NULL, NULL); - ciErrNum |= clEnqueueWriteBuffer(cqCommandQueue[1], cmDevSrcB, CL_FALSE, szHalfBuffer, szHalfBuffer, (void*)&fSourceB[szHalfOffset], 0, NULL, NULL); - shrCheckError(ciErrNum, CL_SUCCESS); - - // Push out the compute for queue 0 and write for queue 1 to the driver - // (not necessary on Linux, Mac OSX or WinXP) - clFlush(cqCommandQueue[0]); - clFlush(cqCommandQueue[1]); - - // Start Phase 2 *********************************** - - // Launch kernel computation, command-queue 1 - ciErrNum = clEnqueueNDRangeKernel(cqCommandQueue[1], ckKernel[1], 1, NULL, &szGlobalWorkSize, &szLocalWorkSize, 0, NULL, NULL); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - - // Non Blocking Read of 1st half of output data from device to host, command-queue 0 - ciErrNum = clEnqueueReadBuffer(cqCommandQueue[0], cmDevResult, CL_FALSE, 0, szHalfBuffer, (void*)&fResult[0], 0, NULL, NULL); - shrCheckError(ciErrNum, CL_SUCCESS); - - // Push out the compute for queue 1 and the read for queue 0 to the driver - // (not necessary on Linux, Mac OSX or WinXP) - clFlush(cqCommandQueue[0]); - clFlush(cqCommandQueue[1]); - - // Start Phase 0 (Rolls over) *********************************** - - // Non Blocking Read of 2nd half of output data from device to host, command-queue 1 - ciErrNum = clEnqueueReadBuffer(cqCommandQueue[1], cmDevResult, CL_FALSE, szHalfBuffer, szHalfBuffer, (void*)&fResult[szHalfOffset], 0, NULL, NULL); - shrCheckError(ciErrNum, CL_SUCCESS); - } - - // *** Sync to host and get average sequence time - clFinish(cqCommandQueue[0]); - clFinish(cqCommandQueue[1]); - dAvgTime = shrDeltaT(0)/(double)iCycles; - - // Log config if asked for - if (bShowConfig) - { - shrLog("\n2-Queue sequence Configuration:\n"); - shrLog(" Global Work Size (per command-queue)\t= %u\n Local Work Size \t\t\t= %u\n # of Work Groups (per command-queue)\t= %u\n # of command-queues\t\t\t= 2\n", - szGlobalWorkSize, szLocalWorkSize, szGlobalWorkSize/szLocalWorkSize); - } - - return dAvgTime; -} - -// Function to adjust compute task according to device capability -// This allows a consistent overlap % across a wide variety of GPU's for test purposes -// It also implitly illustrates the relationship between compute capability and overlap at fixed work size -// ********************************************************************* -int AdjustCompute(cl_device_id cdTargetDevice, unsigned int uiNumElements, int iInitLoopCount, int iCycles) -{ - // Locals - double dCopyTime, dComputeTime; - int iComputedLoopCount; - - // Change Source Data - shrFillArray(fSourceA, (int)uiNumElements); - shrFillArray(fSourceB, (int)uiNumElements); - - // Reset Global work size for 1 command-queue, and log work sizes & dimensions - szGlobalWorkSize = shrRoundUp((int)szLocalWorkSize, (int)(uiNumElements/4)); - - // *** Make sure queues are empty and then start timer - clFinish(cqCommandQueue[0]); - clFinish(cqCommandQueue[1]); - shrDeltaT(0); - - // Run the copy iCycles times and measure copy time on this system - for (int i = 0; i < iCycles; i++) - { - // Nonblocking Write of all of input data from host to device in command-queue 0 - ciErrNum = clEnqueueWriteBuffer(cqCommandQueue[0], cmDevSrcA, CL_FALSE, 0, szBuffBytes, (void*)&fSourceA[0], 0, NULL, NULL); - ciErrNum |= clEnqueueWriteBuffer(cqCommandQueue[0], cmDevSrcB, CL_FALSE, 0, szBuffBytes, (void*)&fSourceB[0], 0, NULL, NULL); - ciErrNum |= clFlush(cqCommandQueue[0]); - shrCheckError(ciErrNum, CL_SUCCESS); - } - clFinish(cqCommandQueue[0]); - dCopyTime = shrDeltaT(0); - - // Run the compute iCycles times and measure compute time on this system - for (int i = 0; i < iCycles; i++) - { - // Launch kernel computation, command-queue 0 - ciErrNum = clEnqueueNDRangeKernel(cqCommandQueue[0], ckKernel[0], 1, NULL, &szGlobalWorkSize, &szLocalWorkSize, 0, NULL, NULL); - ciErrNum |= clFlush(cqCommandQueue[0]); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - } - clFinish(cqCommandQueue[0]); - dComputeTime = shrDeltaT(0); - - // Determine number of core loop cycles proportional to copy/compute time ratio - dComputeTime = MAX(dComputeTime, 1.0e-6); - iComputedLoopCount = CLAMP(2, (int)((dCopyTime/dComputeTime) * (double)iInitLoopCount), (iInitLoopCount * 4)); - ciErrNum |= clSetKernelArg(ckKernel[0], 4, sizeof(cl_int), (void*)&iComputedLoopCount); - ciErrNum |= clSetKernelArg(ckKernel[1], 4, sizeof(cl_int), (void*)&iComputedLoopCount); - oclCheckErrorEX(ciErrNum, CL_SUCCESS, pCleanup); - return (iComputedLoopCount); -} - -// Cleanup/Exit function -// ********************************************************************* -void Cleanup (int iExitCode) -{ - // Cleanup allocated objects - shrLog("Starting Cleanup...\n\n"); - if(cPathAndName)free(cPathAndName); - if(cSourceCL)free(cSourceCL); - if(Golden)free(Golden); - if(ckKernel[0])clReleaseKernel(ckKernel[0]); - if(ckKernel[1])clReleaseKernel(ckKernel[1]); - if(cpProgram)clReleaseProgram(cpProgram); - if(fSourceA)clEnqueueUnmapMemObject(cqCommandQueue[0], cmPinnedSrcA, (void*)fSourceA, 0, NULL, NULL); - if(fSourceB)clEnqueueUnmapMemObject(cqCommandQueue[0], cmPinnedSrcB, (void*)fSourceB, 0, NULL, NULL); - if(fResult)clEnqueueUnmapMemObject(cqCommandQueue[0], cmPinnedResult, (void*)fResult, 0, NULL, NULL); - if(cmDevSrcA)clReleaseMemObject(cmDevSrcA); - if(cmDevSrcB)clReleaseMemObject(cmDevSrcB); - if(cmDevResult)clReleaseMemObject(cmDevResult); - if(cmPinnedSrcA)clReleaseMemObject(cmPinnedSrcA); - if(cmPinnedSrcB)clReleaseMemObject(cmPinnedSrcB); - if(cmPinnedResult)clReleaseMemObject(cmPinnedResult); - if(cqCommandQueue[0])clReleaseCommandQueue(cqCommandQueue[0]); - if(cqCommandQueue[1])clReleaseCommandQueue(cqCommandQueue[1]); - if(cxGPUContext)clReleaseContext(cxGPUContext); - if(cdDevices)free(cdDevices); - - // Master status Pass/Fail (all tests) - shrQAFinishExit( *gp_argc, (const char **)*gp_argv, (iExitCode == EXIT_SUCCESS) ? QA_PASSED : QA_FAILED ); -} - -// "Golden" Host processing vector hyptenuse function for comparison purposes -// ********************************************************************* -void VectorHypotHost(const float* pfData1, const float* pfData2, float* pfResult, unsigned int uiNumElements, int iInnerLoopCount) -{ - for (unsigned int i = 0; i < uiNumElements; i++) - { - float fA = pfData1[i]; - float fB = pfData2[i]; - float fC = sqrtf(fA * fA + fB * fB); - - pfResult[i] = fC; - } -} diff --git a/tests/opencl/vectorhypot/oclUtils.cpp b/tests/opencl/vectorhypot/oclUtils.cpp deleted file mode 100644 index 6d920858..00000000 --- a/tests/opencl/vectorhypot/oclUtils.cpp +++ /dev/null @@ -1,806 +0,0 @@ -/* - * Copyright 1993-2010 NVIDIA Corporation. All rights reserved. - * - * Please refer to the NVIDIA end user license agreement (EULA) associated - * with this source code for terms and conditions that govern your use of - * this software. Any use, reproduction, disclosure, or distribution of - * this software and related documentation outside the terms of the EULA - * is strictly prohibited. - * - */ - -// ********************************************************************* -// Utilities specific to OpenCL samples in NVIDIA GPU Computing SDK -// ********************************************************************* - -#include -#include -#include -#include -#include -#include "oclUtils.h" - -////////////////////////////////////////////////////////////////////////////// -//! Gets the platform ID for NVIDIA if available, otherwise default -//! -//! @return the id -//! @param clSelectedPlatformID OpenCL platoform ID -////////////////////////////////////////////////////////////////////////////// -cl_int oclGetPlatformID(cl_platform_id* clSelectedPlatformID) -{ - char chBuffer[1024]; - cl_uint num_platforms; - cl_platform_id* clPlatformIDs; - cl_int ciErrNum; - *clSelectedPlatformID = NULL; - - // Get OpenCL platform count - ciErrNum = clGetPlatformIDs (0, NULL, &num_platforms); - if (ciErrNum != CL_SUCCESS) - { - shrLog(" Error %i in clGetPlatformIDs Call !!!\n\n", ciErrNum); - return -1000; - } - else - { - if(num_platforms == 0) - { - shrLog("No OpenCL platform found!\n\n"); - return -2000; - } - else - { - // if there's a platform or more, make space for ID's - if ((clPlatformIDs = (cl_platform_id*)malloc(num_platforms * sizeof(cl_platform_id))) == NULL) - { - shrLog("Failed to allocate memory for cl_platform ID's!\n\n"); - return -3000; - } - - // get platform info for each platform and trap the NVIDIA platform if found - ciErrNum = clGetPlatformIDs (num_platforms, clPlatformIDs, NULL); - for(cl_uint i = 0; i < num_platforms; ++i) - { - ciErrNum = clGetPlatformInfo (clPlatformIDs[i], CL_PLATFORM_NAME, 1024, &chBuffer, NULL); - if(ciErrNum == CL_SUCCESS) - { - if(strstr(chBuffer, "NVIDIA") != NULL) - { - *clSelectedPlatformID = clPlatformIDs[i]; - break; - } - } - } - - // default to zeroeth platform if NVIDIA not found - if(*clSelectedPlatformID == NULL) - { - shrLog("WARNING: NVIDIA OpenCL platform not found - defaulting to first platform!\n\n"); - *clSelectedPlatformID = clPlatformIDs[0]; - } - - free(clPlatformIDs); - } - } - - return CL_SUCCESS; -} - -////////////////////////////////////////////////////////////////////////////// -//! Print the device name -//! -//! @param iLogMode enum LOGBOTH, LOGCONSOLE, LOGFILE -//! @param device OpenCL id of the device -////////////////////////////////////////////////////////////////////////////// -void oclPrintDevName(int iLogMode, cl_device_id device) -{ - char device_string[1024]; - clGetDeviceInfo(device, CL_DEVICE_NAME, sizeof(device_string), &device_string, NULL); - shrLogEx(iLogMode, 0, "%s\n", device_string); -} - -////////////////////////////////////////////////////////////////////////////// -//! Print info about the device -//! -//! @param iLogMode enum LOGBOTH, LOGCONSOLE, LOGFILE -//! @param device OpenCL id of the device -////////////////////////////////////////////////////////////////////////////// -void oclPrintDevInfo(int iLogMode, cl_device_id device) -{ - char device_string[1024]; - bool nv_device_attibute_query = false; - - // CL_DEVICE_NAME - clGetDeviceInfo(device, CL_DEVICE_NAME, sizeof(device_string), &device_string, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_NAME: \t\t\t%s\n", device_string); - - // CL_DEVICE_VENDOR - clGetDeviceInfo(device, CL_DEVICE_VENDOR, sizeof(device_string), &device_string, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_VENDOR: \t\t\t%s\n", device_string); - - // CL_DRIVER_VERSION - clGetDeviceInfo(device, CL_DRIVER_VERSION, sizeof(device_string), &device_string, NULL); - shrLogEx(iLogMode, 0, " CL_DRIVER_VERSION: \t\t\t%s\n", device_string); - - // CL_DEVICE_VERSION - clGetDeviceInfo(device, CL_DEVICE_VERSION, sizeof(device_string), &device_string, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_VERSION: \t\t\t%s\n", device_string); - - // CL_DEVICE_OPENCL_C_VERSION (if CL_DEVICE_VERSION version > 1.0) - if(strncmp("OpenCL 1.0", device_string, 10) != 0) - { - // This code is unused for devices reporting OpenCL 1.0, but a def is needed anyway to allow compilation using v 1.0 headers - // This constant isn't #defined in 1.0 - #ifndef CL_DEVICE_OPENCL_C_VERSION - #define CL_DEVICE_OPENCL_C_VERSION 0x103D - #endif - - clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, sizeof(device_string), &device_string, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_OPENCL_C_VERSION: \t\t%s\n", device_string); - } - - // CL_DEVICE_TYPE - cl_device_type type; - clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(type), &type, NULL); - if( type & CL_DEVICE_TYPE_CPU ) - shrLogEx(iLogMode, 0, " CL_DEVICE_TYPE:\t\t\t%s\n", "CL_DEVICE_TYPE_CPU"); - if( type & CL_DEVICE_TYPE_GPU ) - shrLogEx(iLogMode, 0, " CL_DEVICE_TYPE:\t\t\t%s\n", "CL_DEVICE_TYPE_GPU"); - if( type & CL_DEVICE_TYPE_ACCELERATOR ) - shrLogEx(iLogMode, 0, " CL_DEVICE_TYPE:\t\t\t%s\n", "CL_DEVICE_TYPE_ACCELERATOR"); - if( type & CL_DEVICE_TYPE_DEFAULT ) - shrLogEx(iLogMode, 0, " CL_DEVICE_TYPE:\t\t\t%s\n", "CL_DEVICE_TYPE_DEFAULT"); - - // CL_DEVICE_MAX_COMPUTE_UNITS - cl_uint compute_units; - clGetDeviceInfo(device, CL_DEVICE_MAX_COMPUTE_UNITS, sizeof(compute_units), &compute_units, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_MAX_COMPUTE_UNITS:\t\t%u\n", compute_units); - - // CL_DEVICE_MAX_WORK_ITEM_DIMENSIONS - size_t workitem_dims; - clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_ITEM_DIMENSIONS, sizeof(workitem_dims), &workitem_dims, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_MAX_WORK_ITEM_DIMENSIONS:\t%u\n", workitem_dims); - - // CL_DEVICE_MAX_WORK_ITEM_SIZES - size_t workitem_size[3]; - clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_ITEM_SIZES, sizeof(workitem_size), &workitem_size, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_MAX_WORK_ITEM_SIZES:\t%u / %u / %u \n", workitem_size[0], workitem_size[1], workitem_size[2]); - - // CL_DEVICE_MAX_WORK_GROUP_SIZE - size_t workgroup_size; - clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(workgroup_size), &workgroup_size, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_MAX_WORK_GROUP_SIZE:\t%u\n", workgroup_size); - - // CL_DEVICE_MAX_CLOCK_FREQUENCY - cl_uint clock_frequency; - clGetDeviceInfo(device, CL_DEVICE_MAX_CLOCK_FREQUENCY, sizeof(clock_frequency), &clock_frequency, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_MAX_CLOCK_FREQUENCY:\t%u MHz\n", clock_frequency); - - // CL_DEVICE_ADDRESS_BITS - cl_uint addr_bits; - clGetDeviceInfo(device, CL_DEVICE_ADDRESS_BITS, sizeof(addr_bits), &addr_bits, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_ADDRESS_BITS:\t\t%u\n", addr_bits); - - // CL_DEVICE_MAX_MEM_ALLOC_SIZE - cl_ulong max_mem_alloc_size; - clGetDeviceInfo(device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof(max_mem_alloc_size), &max_mem_alloc_size, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_MAX_MEM_ALLOC_SIZE:\t\t%u MByte\n", (unsigned int)(max_mem_alloc_size / (1024 * 1024))); - - // CL_DEVICE_GLOBAL_MEM_SIZE - cl_ulong mem_size; - clGetDeviceInfo(device, CL_DEVICE_GLOBAL_MEM_SIZE, sizeof(mem_size), &mem_size, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_GLOBAL_MEM_SIZE:\t\t%u MByte\n", (unsigned int)(mem_size / (1024 * 1024))); - - // CL_DEVICE_ERROR_CORRECTION_SUPPORT - cl_bool error_correction_support; - clGetDeviceInfo(device, CL_DEVICE_ERROR_CORRECTION_SUPPORT, sizeof(error_correction_support), &error_correction_support, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_ERROR_CORRECTION_SUPPORT:\t%s\n", error_correction_support == CL_TRUE ? "yes" : "no"); - - // CL_DEVICE_LOCAL_MEM_TYPE - cl_device_local_mem_type local_mem_type; - clGetDeviceInfo(device, CL_DEVICE_LOCAL_MEM_TYPE, sizeof(local_mem_type), &local_mem_type, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_LOCAL_MEM_TYPE:\t\t%s\n", local_mem_type == 1 ? "local" : "global"); - - // CL_DEVICE_LOCAL_MEM_SIZE - clGetDeviceInfo(device, CL_DEVICE_LOCAL_MEM_SIZE, sizeof(mem_size), &mem_size, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_LOCAL_MEM_SIZE:\t\t%u KByte\n", (unsigned int)(mem_size / 1024)); - - // CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE - clGetDeviceInfo(device, CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE, sizeof(mem_size), &mem_size, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE:\t%u KByte\n", (unsigned int)(mem_size / 1024)); - - // CL_DEVICE_QUEUE_PROPERTIES - cl_command_queue_properties queue_properties; - clGetDeviceInfo(device, CL_DEVICE_QUEUE_PROPERTIES, sizeof(queue_properties), &queue_properties, NULL); - if( queue_properties & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE ) - shrLogEx(iLogMode, 0, " CL_DEVICE_QUEUE_PROPERTIES:\t\t%s\n", "CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE"); - if( queue_properties & CL_QUEUE_PROFILING_ENABLE ) - shrLogEx(iLogMode, 0, " CL_DEVICE_QUEUE_PROPERTIES:\t\t%s\n", "CL_QUEUE_PROFILING_ENABLE"); - - // CL_DEVICE_IMAGE_SUPPORT - cl_bool image_support; - clGetDeviceInfo(device, CL_DEVICE_IMAGE_SUPPORT, sizeof(image_support), &image_support, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_IMAGE_SUPPORT:\t\t%u\n", image_support); - - // CL_DEVICE_MAX_READ_IMAGE_ARGS - cl_uint max_read_image_args; - clGetDeviceInfo(device, CL_DEVICE_MAX_READ_IMAGE_ARGS, sizeof(max_read_image_args), &max_read_image_args, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_MAX_READ_IMAGE_ARGS:\t%u\n", max_read_image_args); - - // CL_DEVICE_MAX_WRITE_IMAGE_ARGS - cl_uint max_write_image_args; - clGetDeviceInfo(device, CL_DEVICE_MAX_WRITE_IMAGE_ARGS, sizeof(max_write_image_args), &max_write_image_args, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_MAX_WRITE_IMAGE_ARGS:\t%u\n", max_write_image_args); - - // CL_DEVICE_SINGLE_FP_CONFIG - cl_device_fp_config fp_config; - clGetDeviceInfo(device, CL_DEVICE_SINGLE_FP_CONFIG, sizeof(cl_device_fp_config), &fp_config, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_SINGLE_FP_CONFIG:\t\t%s%s%s%s%s%s\n", - fp_config & CL_FP_DENORM ? "denorms " : "", - fp_config & CL_FP_INF_NAN ? "INF-quietNaNs " : "", - fp_config & CL_FP_ROUND_TO_NEAREST ? "round-to-nearest " : "", - fp_config & CL_FP_ROUND_TO_ZERO ? "round-to-zero " : "", - fp_config & CL_FP_ROUND_TO_INF ? "round-to-inf " : "", - fp_config & CL_FP_FMA ? "fma " : ""); - - // CL_DEVICE_IMAGE2D_MAX_WIDTH, CL_DEVICE_IMAGE2D_MAX_HEIGHT, CL_DEVICE_IMAGE3D_MAX_WIDTH, CL_DEVICE_IMAGE3D_MAX_HEIGHT, CL_DEVICE_IMAGE3D_MAX_DEPTH - size_t szMaxDims[5]; - shrLogEx(iLogMode, 0, "\n CL_DEVICE_IMAGE "); - clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_WIDTH, sizeof(size_t), &szMaxDims[0], NULL); - shrLogEx(iLogMode, 0, "\t\t\t2D_MAX_WIDTH\t %u\n", szMaxDims[0]); - clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_HEIGHT, sizeof(size_t), &szMaxDims[1], NULL); - shrLogEx(iLogMode, 0, "\t\t\t\t\t2D_MAX_HEIGHT\t %u\n", szMaxDims[1]); - clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_WIDTH, sizeof(size_t), &szMaxDims[2], NULL); - shrLogEx(iLogMode, 0, "\t\t\t\t\t3D_MAX_WIDTH\t %u\n", szMaxDims[2]); - clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_HEIGHT, sizeof(size_t), &szMaxDims[3], NULL); - shrLogEx(iLogMode, 0, "\t\t\t\t\t3D_MAX_HEIGHT\t %u\n", szMaxDims[3]); - clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_DEPTH, sizeof(size_t), &szMaxDims[4], NULL); - shrLogEx(iLogMode, 0, "\t\t\t\t\t3D_MAX_DEPTH\t %u\n", szMaxDims[4]); - - // CL_DEVICE_EXTENSIONS: get device extensions, and if any then parse & log the string onto separate lines - clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(device_string), &device_string, NULL); - if (device_string != 0) - { - shrLogEx(iLogMode, 0, "\n CL_DEVICE_EXTENSIONS:"); - std::string stdDevString; - stdDevString = std::string(device_string); - size_t szOldPos = 0; - size_t szSpacePos = stdDevString.find(' ', szOldPos); // extensions string is space delimited - while (szSpacePos != stdDevString.npos) - { - if( strcmp("cl_nv_device_attribute_query", stdDevString.substr(szOldPos, szSpacePos - szOldPos).c_str()) == 0 ) - nv_device_attibute_query = true; - - if (szOldPos > 0) - { - shrLogEx(iLogMode, 0, "\t\t"); - } - shrLogEx(iLogMode, 0, "\t\t\t%s\n", stdDevString.substr(szOldPos, szSpacePos - szOldPos).c_str()); - - do { - szOldPos = szSpacePos + 1; - szSpacePos = stdDevString.find(' ', szOldPos); - } while (szSpacePos == szOldPos); - } - shrLogEx(iLogMode, 0, "\n"); - } - else - { - shrLogEx(iLogMode, 0, " CL_DEVICE_EXTENSIONS: None\n"); - } - - if(nv_device_attibute_query) - { - cl_uint compute_capability_major, compute_capability_minor; - clGetDeviceInfo(device, CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV, sizeof(cl_uint), &compute_capability_major, NULL); - clGetDeviceInfo(device, CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV, sizeof(cl_uint), &compute_capability_minor, NULL); - shrLogEx(iLogMode, 0, "\n CL_DEVICE_COMPUTE_CAPABILITY_NV:\t%u.%u\n", compute_capability_major, compute_capability_minor); - - shrLogEx(iLogMode, 0, " NUMBER OF MULTIPROCESSORS:\t\t%u\n", compute_units); // this is the same value reported by CL_DEVICE_MAX_COMPUTE_UNITS - shrLogEx(iLogMode, 0, " NUMBER OF CUDA CORES:\t\t\t%u\n", ConvertSMVer2Cores(compute_capability_major, compute_capability_minor) * compute_units); - - cl_uint regs_per_block; - clGetDeviceInfo(device, CL_DEVICE_REGISTERS_PER_BLOCK_NV, sizeof(cl_uint), ®s_per_block, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_REGISTERS_PER_BLOCK_NV:\t%u\n", regs_per_block); - - cl_uint warp_size; - clGetDeviceInfo(device, CL_DEVICE_WARP_SIZE_NV, sizeof(cl_uint), &warp_size, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_WARP_SIZE_NV:\t\t%u\n", warp_size); - - cl_bool gpu_overlap; - clGetDeviceInfo(device, CL_DEVICE_GPU_OVERLAP_NV, sizeof(cl_bool), &gpu_overlap, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_GPU_OVERLAP_NV:\t\t%s\n", gpu_overlap == CL_TRUE ? "CL_TRUE" : "CL_FALSE"); - - cl_bool exec_timeout; - clGetDeviceInfo(device, CL_DEVICE_KERNEL_EXEC_TIMEOUT_NV, sizeof(cl_bool), &exec_timeout, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_KERNEL_EXEC_TIMEOUT_NV:\t%s\n", exec_timeout == CL_TRUE ? "CL_TRUE" : "CL_FALSE"); - - cl_bool integrated_memory; - clGetDeviceInfo(device, CL_DEVICE_INTEGRATED_MEMORY_NV, sizeof(cl_bool), &integrated_memory, NULL); - shrLogEx(iLogMode, 0, " CL_DEVICE_INTEGRATED_MEMORY_NV:\t%s\n", integrated_memory == CL_TRUE ? "CL_TRUE" : "CL_FALSE"); - } - - // CL_DEVICE_PREFERRED_VECTOR_WIDTH_ - shrLogEx(iLogMode, 0, " CL_DEVICE_PREFERRED_VECTOR_WIDTH_\t"); - cl_uint vec_width [6]; - clGetDeviceInfo(device, CL_DEVICE_PREFERRED_VECTOR_WIDTH_CHAR, sizeof(cl_uint), &vec_width[0], NULL); - clGetDeviceInfo(device, CL_DEVICE_PREFERRED_VECTOR_WIDTH_SHORT, sizeof(cl_uint), &vec_width[1], NULL); - clGetDeviceInfo(device, CL_DEVICE_PREFERRED_VECTOR_WIDTH_INT, sizeof(cl_uint), &vec_width[2], NULL); - clGetDeviceInfo(device, CL_DEVICE_PREFERRED_VECTOR_WIDTH_LONG, sizeof(cl_uint), &vec_width[3], NULL); - clGetDeviceInfo(device, CL_DEVICE_PREFERRED_VECTOR_WIDTH_FLOAT, sizeof(cl_uint), &vec_width[4], NULL); - clGetDeviceInfo(device, CL_DEVICE_PREFERRED_VECTOR_WIDTH_DOUBLE, sizeof(cl_uint), &vec_width[5], NULL); - shrLogEx(iLogMode, 0, "CHAR %u, SHORT %u, INT %u, LONG %u, FLOAT %u, DOUBLE %u\n\n\n", - vec_width[0], vec_width[1], vec_width[2], vec_width[3], vec_width[4], vec_width[5]); -} - -////////////////////////////////////////////////////////////////////////////// -//! Get and return device capability -//! -//! @return the 2 digit integer representation of device Cap (major minor). return -1 if NA -//! @param device OpenCL id of the device -////////////////////////////////////////////////////////////////////////////// -int oclGetDevCap(cl_device_id device) -{ - char cDevString[1024]; - bool bDevAttributeQuery = false; - int iDevArch = -1; - - // Get device extensions, and if any then search for cl_nv_device_attribute_query - clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(cDevString), &cDevString, NULL); - if (cDevString != 0) - { - std::string stdDevString; - stdDevString = std::string(cDevString); - size_t szOldPos = 0; - size_t szSpacePos = stdDevString.find(' ', szOldPos); // extensions string is space delimited - while (szSpacePos != stdDevString.npos) - { - if( strcmp("cl_nv_device_attribute_query", stdDevString.substr(szOldPos, szSpacePos - szOldPos).c_str()) == 0 ) - { - bDevAttributeQuery = true; - } - - do { - szOldPos = szSpacePos + 1; - szSpacePos = stdDevString.find(' ', szOldPos); - } while (szSpacePos == szOldPos); - } - } - - // if search succeeded, get device caps - if(bDevAttributeQuery) - { - cl_int iComputeCapMajor, iComputeCapMinor; - clGetDeviceInfo(device, CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV, sizeof(cl_uint), (void*)&iComputeCapMajor, NULL); - clGetDeviceInfo(device, CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV, sizeof(cl_uint), (void*)&iComputeCapMinor, NULL); - iDevArch = (10 * iComputeCapMajor) + iComputeCapMinor; - } - - return iDevArch; -} - -////////////////////////////////////////////////////////////////////////////// -//! Gets the id of the first device from the context -//! -//! @return the id -//! @param cxGPUContext OpenCL context -////////////////////////////////////////////////////////////////////////////// -cl_device_id oclGetFirstDev(cl_context cxGPUContext) -{ - size_t szParmDataBytes; - cl_device_id* cdDevices; - - // get the list of GPU devices associated with context - clGetContextInfo(cxGPUContext, CL_CONTEXT_DEVICES, 0, NULL, &szParmDataBytes); - cdDevices = (cl_device_id*) malloc(szParmDataBytes); - - clGetContextInfo(cxGPUContext, CL_CONTEXT_DEVICES, szParmDataBytes, cdDevices, NULL); - - cl_device_id first = cdDevices[0]; - free(cdDevices); - - return first; -} - -////////////////////////////////////////////////////////////////////////////// -//! Gets the id of device with maximal FLOPS from the context -//! -//! @return the id -//! @param cxGPUContext OpenCL context -////////////////////////////////////////////////////////////////////////////// -cl_device_id oclGetMaxFlopsDev(cl_context cxGPUContext) -{ - size_t szParmDataBytes; - cl_device_id* cdDevices; - - // get the list of GPU devices associated with context - clGetContextInfo(cxGPUContext, CL_CONTEXT_DEVICES, 0, NULL, &szParmDataBytes); - cdDevices = (cl_device_id*) malloc(szParmDataBytes); - size_t device_count = szParmDataBytes / sizeof(cl_device_id); - - clGetContextInfo(cxGPUContext, CL_CONTEXT_DEVICES, szParmDataBytes, cdDevices, NULL); - - cl_device_id max_flops_device = cdDevices[0]; - int max_flops = 0; - - size_t current_device = 0; - - // CL_DEVICE_MAX_COMPUTE_UNITS - cl_uint compute_units; - clGetDeviceInfo(cdDevices[current_device], CL_DEVICE_MAX_COMPUTE_UNITS, sizeof(compute_units), &compute_units, NULL); - - // CL_DEVICE_MAX_CLOCK_FREQUENCY - cl_uint clock_frequency; - clGetDeviceInfo(cdDevices[current_device], CL_DEVICE_MAX_CLOCK_FREQUENCY, sizeof(clock_frequency), &clock_frequency, NULL); - - max_flops = compute_units * clock_frequency; - ++current_device; - - while( current_device < device_count ) - { - // CL_DEVICE_MAX_COMPUTE_UNITS - cl_uint compute_units; - clGetDeviceInfo(cdDevices[current_device], CL_DEVICE_MAX_COMPUTE_UNITS, sizeof(compute_units), &compute_units, NULL); - - // CL_DEVICE_MAX_CLOCK_FREQUENCY - cl_uint clock_frequency; - clGetDeviceInfo(cdDevices[current_device], CL_DEVICE_MAX_CLOCK_FREQUENCY, sizeof(clock_frequency), &clock_frequency, NULL); - - int flops = compute_units * clock_frequency; - if( flops > max_flops ) - { - max_flops = flops; - max_flops_device = cdDevices[current_device]; - } - ++current_device; - } - - free(cdDevices); - - return max_flops_device; -} - -////////////////////////////////////////////////////////////////////////////// -//! Loads a Program file and prepends the cPreamble to the code. -//! -//! @return the source string if succeeded, 0 otherwise -//! @param cFilename program filename -//! @param cPreamble code that is prepended to the loaded file, typically a set of #defines or a header -//! @param szFinalLength returned length of the code string -////////////////////////////////////////////////////////////////////////////// -char* oclLoadProgSource(const char* cFilename, const char* cPreamble, size_t* szFinalLength) -{ - // locals - FILE* pFileStream = NULL; - size_t szSourceLength; - - // open the OpenCL source code file - #ifdef _WIN32 // Windows version - if(fopen_s(&pFileStream, cFilename, "rb") != 0) - { - return NULL; - } - #else // Linux version - pFileStream = fopen(cFilename, "rb"); - if(pFileStream == 0) - { - return NULL; - } - #endif - - size_t szPreambleLength = strlen(cPreamble); - - // get the length of the source code - fseek(pFileStream, 0, SEEK_END); - szSourceLength = ftell(pFileStream); - fseek(pFileStream, 0, SEEK_SET); - - // allocate a buffer for the source code string and read it in - char* cSourceString = (char *)malloc(szSourceLength + szPreambleLength + 1); - memcpy(cSourceString, cPreamble, szPreambleLength); - if (fread((cSourceString) + szPreambleLength, szSourceLength, 1, pFileStream) != 1) - { - fclose(pFileStream); - free(cSourceString); - return 0; - } - - // close the file and return the total length of the combined (preamble + source) string - fclose(pFileStream); - if(szFinalLength != 0) - { - *szFinalLength = szSourceLength + szPreambleLength; - } - cSourceString[szSourceLength + szPreambleLength] = '\0'; - - return cSourceString; -} - -////////////////////////////////////////////////////////////////////////////// -//! Gets the id of the nth device from the context -//! -//! @return the id or -1 when out of range -//! @param cxGPUContext OpenCL context -//! @param device_idx index of the device of interest -////////////////////////////////////////////////////////////////////////////// -cl_device_id oclGetDev(cl_context cxGPUContext, unsigned int nr) -{ - size_t szParmDataBytes; - cl_device_id* cdDevices; - - // get the list of GPU devices associated with context - clGetContextInfo(cxGPUContext, CL_CONTEXT_DEVICES, 0, NULL, &szParmDataBytes); - - if( szParmDataBytes / sizeof(cl_device_id) <= nr ) { - return (cl_device_id)-1; - } - - cdDevices = (cl_device_id*) malloc(szParmDataBytes); - - clGetContextInfo(cxGPUContext, CL_CONTEXT_DEVICES, szParmDataBytes, cdDevices, NULL); - - cl_device_id device = cdDevices[nr]; - free(cdDevices); - - return device; -} - -////////////////////////////////////////////////////////////////////////////// -//! Get the binary (PTX) of the program associated with the device -//! -//! @param cpProgram OpenCL program -//! @param cdDevice device of interest -//! @param binary returned code -//! @param length length of returned code -////////////////////////////////////////////////////////////////////////////// -void oclGetProgBinary( cl_program cpProgram, cl_device_id cdDevice, char** binary, size_t* length) -{ - // Grab the number of devices associated witht the program - cl_uint num_devices; - clGetProgramInfo(cpProgram, CL_PROGRAM_NUM_DEVICES, sizeof(cl_uint), &num_devices, NULL); - - // Grab the device ids - cl_device_id* devices = (cl_device_id*) malloc(num_devices * sizeof(cl_device_id)); - clGetProgramInfo(cpProgram, CL_PROGRAM_DEVICES, num_devices * sizeof(cl_device_id), devices, 0); - - // Grab the sizes of the binaries - size_t* binary_sizes = (size_t*)malloc(num_devices * sizeof(size_t)); - clGetProgramInfo(cpProgram, CL_PROGRAM_BINARY_SIZES, num_devices * sizeof(size_t), binary_sizes, NULL); - - // Now get the binaries - char** ptx_code = (char**) malloc(num_devices * sizeof(char*)); - for( unsigned int i=0; i= 0 && index < errorCount) ? errorString[index] : "Unspecified Error"; -} - -// Helper function to get OpenCL image format string (channel order and type) from constant -// ********************************************************************* -const char* oclImageFormatString(cl_uint uiImageFormat) -{ - // cl_channel_order - if (uiImageFormat == CL_R)return "CL_R"; - if (uiImageFormat == CL_A)return "CL_A"; - if (uiImageFormat == CL_RG)return "CL_RG"; - if (uiImageFormat == CL_RA)return "CL_RA"; - if (uiImageFormat == CL_RGB)return "CL_RGB"; - if (uiImageFormat == CL_RGBA)return "CL_RGBA"; - if (uiImageFormat == CL_BGRA)return "CL_BGRA"; - if (uiImageFormat == CL_ARGB)return "CL_ARGB"; - if (uiImageFormat == CL_INTENSITY)return "CL_INTENSITY"; - if (uiImageFormat == CL_LUMINANCE)return "CL_LUMINANCE"; - - // cl_channel_type - if (uiImageFormat == CL_SNORM_INT8)return "CL_SNORM_INT8"; - if (uiImageFormat == CL_SNORM_INT16)return "CL_SNORM_INT16"; - if (uiImageFormat == CL_UNORM_INT8)return "CL_UNORM_INT8"; - if (uiImageFormat == CL_UNORM_INT16)return "CL_UNORM_INT16"; - if (uiImageFormat == CL_UNORM_SHORT_565)return "CL_UNORM_SHORT_565"; - if (uiImageFormat == CL_UNORM_SHORT_555)return "CL_UNORM_SHORT_555"; - if (uiImageFormat == CL_UNORM_INT_101010)return "CL_UNORM_INT_101010"; - if (uiImageFormat == CL_SIGNED_INT8)return "CL_SIGNED_INT8"; - if (uiImageFormat == CL_SIGNED_INT16)return "CL_SIGNED_INT16"; - if (uiImageFormat == CL_SIGNED_INT32)return "CL_SIGNED_INT32"; - if (uiImageFormat == CL_UNSIGNED_INT8)return "CL_UNSIGNED_INT8"; - if (uiImageFormat == CL_UNSIGNED_INT16)return "CL_UNSIGNED_INT16"; - if (uiImageFormat == CL_UNSIGNED_INT32)return "CL_UNSIGNED_INT32"; - if (uiImageFormat == CL_HALF_FLOAT)return "CL_HALF_FLOAT"; - if (uiImageFormat == CL_FLOAT)return "CL_FLOAT"; - - // unknown constant - return "Unknown"; -} diff --git a/tests/opencl/vectorhypot/oclUtils.h b/tests/opencl/vectorhypot/oclUtils.h deleted file mode 100644 index 2b109e18..00000000 --- a/tests/opencl/vectorhypot/oclUtils.h +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright 1993-2010 NVIDIA Corporation. All rights reserved. - * - * Please refer to the NVIDIA end user license agreement (EULA) associated - * with this source code for terms and conditions that govern your use of - * this software. Any use, reproduction, disclosure, or distribution of - * this software and related documentation outside the terms of the EULA - * is strictly prohibited. - * - */ - -#ifndef OCL_UTILS_H -#define OCL_UTILS_H - -// ********************************************************************* -// Utilities specific to OpenCL samples in NVIDIA GPU Computing SDK -// ********************************************************************* - -// Common headers: Cross-API utililties and OpenCL header -#include - -// All OpenCL headers -#if defined (__APPLE__) || defined(MACOSX) - #include -#else - #include -#endif - -// Includes -#include -#include -#include - -// For systems with CL_EXT that are not updated with these extensions, we copied these -// extensions from -#ifndef CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV - /* cl_nv_device_attribute_query extension - no extension #define since it has no functions */ - #define CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV 0x4000 - #define CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV 0x4001 - #define CL_DEVICE_REGISTERS_PER_BLOCK_NV 0x4002 - #define CL_DEVICE_WARP_SIZE_NV 0x4003 - #define CL_DEVICE_GPU_OVERLAP_NV 0x4004 - #define CL_DEVICE_KERNEL_EXEC_TIMEOUT_NV 0x4005 - #define CL_DEVICE_INTEGRATED_MEMORY_NV 0x4006 -#endif - -// reminders for build output window and log -#ifdef _WIN32 - #pragma message ("Note: including shrUtils.h") - #pragma message ("Note: including opencl.h") -#endif - -// SDK Revision # -#define OCL_SDKREVISION "7027912" - -// Error and Exit Handling Macros... -// ********************************************************************* -// Full error handling macro with Cleanup() callback (if supplied)... -// (Companion Inline Function lower on page) -#define oclCheckErrorEX(a, b, c) __oclCheckErrorEX(a, b, c, __FILE__ , __LINE__) - -// Short version without Cleanup() callback pointer -// Both Input (a) and Reference (b) are specified as args -#define oclCheckError(a, b) oclCheckErrorEX(a, b, 0) - -////////////////////////////////////////////////////////////////////////////// -//! Gets the platform ID for NVIDIA if available, otherwise default to platform 0 -//! -//! @return the id -//! @param clSelectedPlatformID OpenCL platform ID -////////////////////////////////////////////////////////////////////////////// -extern "C" cl_int oclGetPlatformID(cl_platform_id* clSelectedPlatformID); - -////////////////////////////////////////////////////////////////////////////// -//! Print info about the device -//! -//! @param iLogMode enum LOGBOTH, LOGCONSOLE, LOGFILE -//! @param device OpenCL id of the device -////////////////////////////////////////////////////////////////////////////// -extern "C" void oclPrintDevInfo(int iLogMode, cl_device_id device); - -////////////////////////////////////////////////////////////////////////////// -//! Get and return device capability -//! -//! @return the 2 digit integer representation of device Cap (major minor). return -1 if NA -//! @param device OpenCL id of the device -////////////////////////////////////////////////////////////////////////////// -extern "C" int oclGetDevCap(cl_device_id device); - -////////////////////////////////////////////////////////////////////////////// -//! Print the device name -//! -//! @param iLogMode enum LOGBOTH, LOGCONSOLE, LOGFILE -//! @param device OpenCL id of the device -////////////////////////////////////////////////////////////////////////////// -extern "C" void oclPrintDevName(int iLogMode, cl_device_id device); - -////////////////////////////////////////////////////////////////////////////// -//! Gets the id of the first device from the context -//! -//! @return the id -//! @param cxGPUContext OpenCL context -////////////////////////////////////////////////////////////////////////////// -extern "C" cl_device_id oclGetFirstDev(cl_context cxGPUContext); - -////////////////////////////////////////////////////////////////////////////// -//! Gets the id of the nth device from the context -//! -//! @return the id or -1 when out of range -//! @param cxGPUContext OpenCL context -//! @param device_idx index of the device of interest -////////////////////////////////////////////////////////////////////////////// -extern "C" cl_device_id oclGetDev(cl_context cxGPUContext, unsigned int device_idx); - -////////////////////////////////////////////////////////////////////////////// -//! Gets the id of device with maximal FLOPS from the context -//! -//! @return the id -//! @param cxGPUContext OpenCL context -////////////////////////////////////////////////////////////////////////////// -extern "C" cl_device_id oclGetMaxFlopsDev(cl_context cxGPUContext); - -////////////////////////////////////////////////////////////////////////////// -//! Loads a Program file and prepends the cPreamble to the code. -//! -//! @return the source string if succeeded, 0 otherwise -//! @param cFilename program filename -//! @param cPreamble code that is prepended to the loaded file, typically a set of #defines or a header -//! @param szFinalLength returned length of the code string -////////////////////////////////////////////////////////////////////////////// -extern "C" char* oclLoadProgSource(const char* cFilename, const char* cPreamble, size_t* szFinalLength); - -////////////////////////////////////////////////////////////////////////////// -//! Get the binary (PTX) of the program associated with the device -//! -//! @param cpProgram OpenCL program -//! @param cdDevice device of interest -//! @param binary returned code -//! @param length length of returned code -////////////////////////////////////////////////////////////////////////////// -extern "C" void oclGetProgBinary( cl_program cpProgram, cl_device_id cdDevice, char** binary, size_t* length); - -////////////////////////////////////////////////////////////////////////////// -//! Get and log the binary (PTX) from the OpenCL compiler for the requested program & device -//! -//! @param cpProgram OpenCL program -//! @param cdDevice device of interest -//! @param const char* cPtxFileName optional PTX file name -////////////////////////////////////////////////////////////////////////////// -extern "C" void oclLogPtx(cl_program cpProgram, cl_device_id cdDevice, const char* cPtxFileName); - -////////////////////////////////////////////////////////////////////////////// -//! Get and log the Build Log from the OpenCL compiler for the requested program & device -//! -//! @param cpProgram OpenCL program -//! @param cdDevice device of interest -////////////////////////////////////////////////////////////////////////////// -extern "C" void oclLogBuildInfo(cl_program cpProgram, cl_device_id cdDevice); - -// Helper function for De-allocating cl objects -// ********************************************************************* -extern "C" void oclDeleteMemObjs(cl_mem* cmMemObjs, int iNumObjs); - -// Helper function to get OpenCL error string from constant -// ********************************************************************* -extern "C" const char* oclErrorString(cl_int error); - -// Helper function to get OpenCL image format string (channel order and type) from constant -// ********************************************************************* -extern "C" const char* oclImageFormatString(cl_uint uiImageFormat); - -// companion inline function for error checking and exit on error WITH Cleanup Callback (if supplied) -// ********************************************************************* -inline void __oclCheckErrorEX(cl_int iSample, cl_int iReference, void (*pCleanup)(int), const char* cFile, const int iLine) -{ - // An error condition is defined by the sample/test value not equal to the reference - if (iReference != iSample) - { - // If the sample/test value isn't equal to the ref, it's an error by defnition, so override 0 sample/test value - iSample = (iSample == 0) ? -9999 : iSample; - - // Log the error info - shrLog("\n !!! Error # %i (%s) at line %i , in file %s !!!\n\n", iSample, oclErrorString(iSample), iLine, cFile); - - // Cleanup and exit, or just exit if no cleanup function pointer provided. Use iSample (error code in this case) as process exit code. - if (pCleanup != NULL) - { - pCleanup(iSample); - } - else - { - shrLogEx(LOGBOTH | CLOSELOG, 0, "Exiting...\n"); - exit(iSample); - } - } -} - -#endif \ No newline at end of file diff --git a/tests/opencl/vectorhypot/shrQATest.h b/tests/opencl/vectorhypot/shrQATest.h deleted file mode 100644 index 245cf8dc..00000000 --- a/tests/opencl/vectorhypot/shrQATest.h +++ /dev/null @@ -1,238 +0,0 @@ -/* -* Copyright 1993-2010 NVIDIA Corporation. All rights reserved. -* -* Please refer to the NVIDIA end user license agreement (EULA) associated -* with this source code for terms and conditions that govern your use of -* this software. Any use, reproduction, disclosure, or distribution of -* this software and related documentation outside the terms of the EULA -* is strictly prohibited. -* -*/ - -#ifndef SHR_QATEST_H -#define SHR_QATEST_H - -// ********************************************************************* -// Generic utilities for NVIDIA GPU Computing SDK -// ********************************************************************* - -// OS dependent includes -#ifdef _WIN32 - #pragma message ("Note: including windows.h") - #pragma message ("Note: including math.h") - #pragma message ("Note: including assert.h") - #pragma message ("Note: including time.h") - -// Headers needed for Windows - #include - #include -#else - // Headers needed for Linux - #include - #include - #include - #include - #include - #include - #include - #include - #include -#endif - -#ifndef STRCASECMP -#ifdef _WIN32 -#define STRCASECMP _stricmp -#else -#define STRCASECMP strcasecmp -#endif -#endif - -#ifndef STRNCASECMP -#ifdef _WIN32 -#define STRNCASECMP _strnicmp -#else -#define STRNCASECMP strncasecmp -#endif -#endif - - -// Standardized QA Start/Finish for CUDA SDK tests -#define shrQAStart(a, b) __shrQAStart(a, b) -#define shrQAFinish(a, b, c) __shrQAFinish(a, b, c) -#define shrQAFinish2(a, b, c, d) __shrQAFinish2(a, b, c, d) - -inline int findExeNameStart(const char *exec_name) -{ - int exename_start = (int)strlen(exec_name); - - while( (exename_start > 0) && - (exec_name[exename_start] != '\\') && - (exec_name[exename_start] != '/') ) - { - exename_start--; - } - if (exec_name[exename_start] == '\\' || - exec_name[exename_start] == '/') - { - return exename_start+1; - } else { - return exename_start; - } -} - -inline int __shrQAStart(int argc, char **argv) -{ - bool bQATest = false; - // First clear the output buffer - fflush(stdout); - fflush(stdout); - - for (int i=1; i < argc; i++) { - int string_start = 0; - while (argv[i][string_start] == '-') - string_start++; - char *string_argv = &argv[i][string_start]; - - if (!STRCASECMP(string_argv, "qatest")) { - bQATest = true; - } - } - - // We don't want to print the entire path, so we search for the first - int exename_start = findExeNameStart(argv[0]); - if (bQATest) { - fprintf(stdout, "&&&& RUNNING %s", &(argv[0][exename_start])); - for (int i=1; i < argc; i++) fprintf(stdout, " %s", argv[i]); - fprintf(stdout, "\n"); - } else { - fprintf(stdout, "[%s] starting...\n", &(argv[0][exename_start])); - } - fflush(stdout); - printf("\n"); fflush(stdout); - return exename_start; -} - -enum eQAstatus { - QA_FAILED = 0, - QA_PASSED = 1, - QA_WAIVED = 2 -}; - -inline void __ExitInTime(int seconds) -{ - fprintf(stdout, "> exiting in %d seconds: ", seconds); - fflush(stdout); - time_t t; - int count; - for (t=time(0)+seconds, count=seconds; time(0) < t; count--) { - fprintf(stdout, "%d...", count); -#ifdef WIN32 - Sleep(1000); -#else - sleep(1); -#endif - } - fprintf(stdout,"done!\n\n"); - fflush(stdout); -} - - -inline void __shrQAFinish(int argc, const char **argv, int iStatus) -{ - // By default QATest is disabled and NoPrompt is Enabled (times out at seconds passed into __ExitInTime() ) - bool bQATest = false, bNoPrompt = true, bQuitInTime = true; - const char *sStatus[] = { "FAILED", "PASSED", "WAIVED", NULL }; - - for (int i=1; i < argc; i++) { - int string_start = 0; - while (argv[i][string_start] == '-') - string_start++; - - const char *string_argv = &argv[i][string_start]; - if (!STRCASECMP(string_argv, "qatest")) { - bQATest = true; - } - // For SDK individual samples that don't specify -noprompt or -prompt, - // a 3 second delay will happen before exiting, giving a user time to view results - if (!STRCASECMP(string_argv, "noprompt") || !STRCASECMP(string_argv, "help")) { - bNoPrompt = true; - bQuitInTime = false; - } - if (!STRCASECMP(string_argv, "prompt")) { - bNoPrompt = false; - bQuitInTime = false; - } - } - - int exename_start = findExeNameStart(argv[0]); - if (bQATest) { - fprintf(stdout, "&&&& %s %s", sStatus[iStatus], &(argv[0][exename_start])); - for (int i=1; i < argc; i++) fprintf(stdout, " %s", argv[i]); - fprintf(stdout, "\n"); - } else { - fprintf(stdout, "[%s] test results...\n%s\n", &(argv[0][exename_start]), sStatus[iStatus]); - } - fflush(stdout); - printf("\n"); fflush(stdout); - if (bQuitInTime) { - __ExitInTime(3); - } else { - if (!bNoPrompt) { - fprintf(stdout, "\nPress to exit...\n"); - fflush(stdout); - getchar(); - } - } -} - -inline void __shrQAFinish2(bool bQATest, int argc, const char **argv, int iStatus) -{ - bool bQuitInTime = true; - const char *sStatus[] = { "FAILED", "PASSED", "WAIVED", NULL }; - - for (int i=1; i < argc; i++) { - int string_start = 0; - while (argv[i][string_start] == '-') - string_start++; - - const char *string_argv = &argv[i][string_start]; - // For SDK individual samples that don't specify -noprompt or -prompt, - // a 3 second delay will happen before exiting, giving a user time to view results - if (!STRCASECMP(string_argv, "noprompt") || !STRCASECMP(string_argv, "help")) { - bQuitInTime = false; - } - if (!STRCASECMP(string_argv, "prompt")) { - bQuitInTime = false; - } - } - - int exename_start = findExeNameStart(argv[0]); - if (bQATest) { - fprintf(stdout, "&&&& %s %s", sStatus[iStatus], &(argv[0][exename_start])); - for (int i=1; i < argc; i++) fprintf(stdout, " %s", argv[i]); - fprintf(stdout, "\n"); - } else { - fprintf(stdout, "[%s] test results...\n%s\n", &(argv[0][exename_start]), sStatus[iStatus]); - } - fflush(stdout); - - if (bQuitInTime) { - __ExitInTime(3); - } -} - -inline void shrQAFinishExit(int argc, const char **argv, int iStatus) -{ - __shrQAFinish(argc, argv, iStatus); - - exit(iStatus ? EXIT_SUCCESS : EXIT_FAILURE); -} - -inline void shrQAFinishExit2(bool bQAtest, int argc, const char **argv, int iStatus) -{ - __shrQAFinish2(bQAtest, argc, argv, iStatus); - - exit(iStatus ? EXIT_SUCCESS : EXIT_FAILURE); -} - -#endif \ No newline at end of file diff --git a/tests/opencl/vectorhypot/shrUtils.cpp b/tests/opencl/vectorhypot/shrUtils.cpp deleted file mode 100644 index cf0d2c3e..00000000 --- a/tests/opencl/vectorhypot/shrUtils.cpp +++ /dev/null @@ -1,1954 +0,0 @@ -/* - * Copyright 1993-2010 NVIDIA Corporation. All rights reserved. - * - * Please refer to the NVIDIA end user license agreement (EULA) associated - * with this source code for terms and conditions that govern your use of - * this software. Any use, reproduction, disclosure, or distribution of - * this software and related documentation outside the terms of the EULA - * is strictly prohibited. - * - */ - -// ********************************************************************* -// Generic Utilities for NVIDIA GPU Computing SDK -// ********************************************************************* - -// includes -#include -#include -#include -#include -#include "shrUtils.h" -#include "cmd_arg_reader.h" - -// size of PGM file header -const unsigned int PGMHeaderSize = 0x40; -#define MIN_EPSILON_ERROR 1e-3f - -// Deallocate memory allocated within shrUtils -// ********************************************************************* -void shrFree(void* ptr) -{ - if( NULL != ptr) free( ptr); -} - -// Helper function to init data arrays -// ********************************************************************* -void shrFillArray(float* pfData, int iSize) -{ - int i; - const float fScale = 1.0f / (float)RAND_MAX; - for (i = 0; i < iSize; ++i) - { - pfData[i] = fScale * rand(); - } -} - -// Helper function to print data arrays -// ********************************************************************* -void shrPrintArray(float* pfData, int iSize) -{ - int i; - for (i = 0; i < iSize; ++i) - { - shrLog("%d: %.3f\n", i, pfData[i]); - } -} - -// Helper function to return precision delta time for 3 counters since last call based upon host high performance counter -// ********************************************************************* -double shrDeltaT(int iCounterID = 0) -{ - // local var for computation of microseconds since last call - double DeltaT; - - #ifdef _WIN32 // Windows version of precision host timer - - // Variables that need to retain state between calls - static LARGE_INTEGER liOldCount0 = {0, 0}; - static LARGE_INTEGER liOldCount1 = {0, 0}; - static LARGE_INTEGER liOldCount2 = {0, 0}; - - // locals for new count, new freq and new time delta - LARGE_INTEGER liNewCount, liFreq; - if (QueryPerformanceFrequency(&liFreq)) - { - // Get new counter reading - QueryPerformanceCounter(&liNewCount); - - // Update the requested timer - switch (iCounterID) - { - case 0: - { - // Calculate time difference for timer 0. (zero when called the first time) - DeltaT = liOldCount0.LowPart ? (((double)liNewCount.QuadPart - (double)liOldCount0.QuadPart) / (double)liFreq.QuadPart) : 0.0; - - // Reset old count to new - liOldCount0 = liNewCount; - - break; - } - case 1: - { - // Calculate time difference for timer 1. (zero when called the first time) - DeltaT = liOldCount1.LowPart ? (((double)liNewCount.QuadPart - (double)liOldCount1.QuadPart) / (double)liFreq.QuadPart) : 0.0; - - // Reset old count to new - liOldCount1 = liNewCount; - - break; - } - case 2: - { - // Calculate time difference for timer 2. (zero when called the first time) - DeltaT = liOldCount2.LowPart ? (((double)liNewCount.QuadPart - (double)liOldCount2.QuadPart) / (double)liFreq.QuadPart) : 0.0; - - // Reset old count to new - liOldCount2 = liNewCount; - - break; - } - default: - { - // Requested counter ID out of range - return -9999.0; - } - } - - // Returns time difference in seconds sunce the last call - return DeltaT; - } - else - { - // No high resolution performance counter - return -9999.0; - } - #else // Linux version of precision host timer. See http://www.informit.com/articles/article.aspx?p=23618&seqNum=8 - static struct timeval _NewTime; // new wall clock time (struct representation in seconds and microseconds) - static struct timeval _OldTime0; // old wall clock time 0(struct representation in seconds and microseconds) - static struct timeval _OldTime1; // old wall clock time 1(struct representation in seconds and microseconds) - static struct timeval _OldTime2; // old wall clock time 2(struct representation in seconds and microseconds) - - // Get new counter reading - gettimeofday(&_NewTime, NULL); - - switch (iCounterID) - { - case 0: - { - // Calculate time difference for timer 0. (zero when called the first time) - DeltaT = ((double)_NewTime.tv_sec + 1.0e-6 * (double)_NewTime.tv_usec) - ((double)_OldTime0.tv_sec + 1.0e-6 * (double)_OldTime0.tv_usec); - - // Reset old time 0 to new - _OldTime0.tv_sec = _NewTime.tv_sec; - _OldTime0.tv_usec = _NewTime.tv_usec; - - break; - } - case 1: - { - // Calculate time difference for timer 1. (zero when called the first time) - DeltaT = ((double)_NewTime.tv_sec + 1.0e-6 * (double)_NewTime.tv_usec) - ((double)_OldTime1.tv_sec + 1.0e-6 * (double)_OldTime1.tv_usec); - - // Reset old time 1 to new - _OldTime1.tv_sec = _NewTime.tv_sec; - _OldTime1.tv_usec = _NewTime.tv_usec; - - break; - } - case 2: - { - // Calculate time difference for timer 2. (zero when called the first time) - DeltaT = ((double)_NewTime.tv_sec + 1.0e-6 * (double)_NewTime.tv_usec) - ((double)_OldTime2.tv_sec + 1.0e-6 * (double)_OldTime2.tv_usec); - - // Reset old time 2 to new - _OldTime2.tv_sec = _NewTime.tv_sec; - _OldTime2.tv_usec = _NewTime.tv_usec; - - break; - } - default: - { - // Requested counter ID out of range - return -9999.0; - } - } - - // Returns time difference in seconds sunce the last call - return DeltaT; - #endif -} - -// Optional LogFileName Override function -// ********************************************************************* -char* cLogFilePathAndName = NULL; -void shrSetLogFileName (const char* cOverRideName) -{ - if( cLogFilePathAndName != NULL ) { - free(cLogFilePathAndName); - } - cLogFilePathAndName = (char*) malloc(strlen(cOverRideName) + 1); - #ifdef WIN32 - strcpy_s(cLogFilePathAndName, strlen(cOverRideName) + 1, cOverRideName); - #else - strcpy(cLogFilePathAndName, cOverRideName); - #endif - return; -} - -// Function to log standardized information to console, file or both -// ********************************************************************* -static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list vaArgList) -{ - static FILE* pFileStream0 = NULL; - static FILE* pFileStream1 = NULL; - size_t szNumWritten = 0; - char cFileMode [3]; - - // if the sample log file is closed and the call incudes a "write-to-file", open file for writing - if ((pFileStream0 == NULL) && (iLogMode & LOGFILE)) - { - // if the default filename has not been overriden, set to default - if (cLogFilePathAndName == NULL) - { - shrSetLogFileName(DEFAULTLOGFILE); - } - - #ifdef _WIN32 // Windows version - // set the file mode - if (iLogMode & APPENDMODE) // append to prexisting file contents - { - sprintf_s (cFileMode, 3, "a+"); - } - else // replace prexisting file contents - { - sprintf_s (cFileMode, 3, "w"); - } - - // open the individual sample log file in the requested mode - errno_t err = fopen_s(&pFileStream0, cLogFilePathAndName, cFileMode); - - // if error on attempt to open, be sure the file is null or close it, then return negative error code - if (err != 0) - { - if (pFileStream0) - { - fclose (pFileStream0); - } - return -err; - } - #else // Linux & Mac version - // set the file mode - if (iLogMode & APPENDMODE) // append to prexisting file contents - { - sprintf (cFileMode, "a+"); - } - else // replace prexisting file contents - { - sprintf (cFileMode, "w"); - } - - // open the file in the requested mode - if ((pFileStream0 = fopen(cLogFilePathAndName, cFileMode)) == 0) - { - // if error on attempt to open, be sure the file is null or close it, then return negative error code - if (pFileStream0) - { - fclose (pFileStream0); - } - return -1; - } - #endif - } - - // if the master log file is closed and the call incudes a "write-to-file" and MASTER, open master logfile file for writing - if ((pFileStream1 == NULL) && (iLogMode & LOGFILE) && (iLogMode & MASTER)) - { - #ifdef _WIN32 // Windows version - // open the master log file in append mode - errno_t err = fopen_s(&pFileStream1, MASTERLOGFILE, "a+"); - - // if error on attempt to open, be sure the file is null or close it, then return negative error code - if (err != 0) - { - if (pFileStream1) - { - fclose (pFileStream1); - pFileStream1 = NULL; - } - iLogMode = LOGCONSOLE; // Force to LOGCONSOLE only since the file stream is invalid -// return -err; - } - #else // Linux & Mac version - - // open the file in the requested mode - if ((pFileStream1 = fopen(MASTERLOGFILE, "a+")) == 0) - { - // if error on attempt to open, be sure the file is null or close it, then return negative error code - if (pFileStream1) - { - fclose (pFileStream1); - pFileStream1 = NULL; - } - iLogMode = LOGCONSOLE; // Force to LOGCONSOLE only since the file stream is invalid -// return -1; - } - #endif - - // If master log file length has become excessive, empty/reopen - if (iLogMode != LOGCONSOLE) - { - fseek(pFileStream1, 0L, SEEK_END); - if (ftell(pFileStream1) > 50000L) - { - fclose (pFileStream1); - #ifdef _WIN32 // Windows version - fopen_s(&pFileStream1, MASTERLOGFILE, "w"); - #else - pFileStream1 = fopen(MASTERLOGFILE, "w"); - #endif - } - } - } - - // Handle special Error Message code - if (iLogMode & ERRORMSG) - { - // print string to console if flagged - if (iLogMode & LOGCONSOLE) - { - szNumWritten = printf ("\n !!! Error # %i at ", iErrNum); // console - } - // print string to file if flagged - if (iLogMode & LOGFILE) - { - szNumWritten = fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file - } - } - - // Vars used for variable argument processing - const char* pStr; - const char* cArg; - int iArg; - double dArg; - unsigned int uiArg; - std::string sFormatSpec; - const std::string sFormatChars = " -+#0123456789.dioufnpcsXxEeGgAa"; - const std::string sTypeChars = "dioufnpcsXxEeGgAa"; - char cType = 'c'; - - // Start at the head of the string and scan to the null at the end - for (pStr = cFormatString; *pStr; ++pStr) - { - // Check if the current character is not a formatting specifier ('%') - if (*pStr != '%') - { - // character is not '%', so print it verbatim to console and/or files as flagged - if (iLogMode & LOGCONSOLE) - { - szNumWritten = putc(*pStr, stdout); // console - } - if (iLogMode & LOGFILE) - { - szNumWritten = putc(*pStr, pFileStream0); // sample log file - if (iLogMode & MASTER) - { - szNumWritten = putc(*pStr, pFileStream1); // master log file - } - } - } - else - { - // character is '%', so skip over it and read the full format specifier for the argument - ++pStr; - sFormatSpec = '%'; - - // special handling for string of %%%% - bool bRepeater = (*pStr == '%'); - if (bRepeater) - { - cType = '%'; - } - - // chars after the '%' are part of format if on list of constants... scan until that isn't true or NULL is found - while (pStr && ((sFormatChars.find(*pStr) != std::string::npos) || bRepeater)) - { - sFormatSpec += *pStr; - - // If the char is a type specifier, trap it and stop scanning - // (a type specifier char is always the last in the format except for string of %%%) - if (sTypeChars.find(*pStr) != std::string::npos) - { - cType = *pStr; - break; - } - - // Special handling for string of %%% - // If a string of %%% was started and then it ends, break (There won't be a typical type specifier) - if (bRepeater && (*pStr != '%')) - { - break; - } - - pStr++; - } - - // Now handle the arg according to type - switch (cType) - { - case '%': // special handling for string of %%%% - { - if (iLogMode & LOGCONSOLE) - { - szNumWritten = printf(sFormatSpec.c_str()); // console - } - if (iLogMode & LOGFILE) - { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str()); // sample log file - if (iLogMode & MASTER) - { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str()); // master log file - } - } - continue; - } - case 'c': // single byte char - case 's': // string of single byte chars - { - // Set cArg as the next value in list and print to console and/or files if flagged - cArg = va_arg(vaArgList, char*); - if (iLogMode & LOGCONSOLE) - { - szNumWritten = printf(sFormatSpec.c_str(), cArg); // console - } - if (iLogMode & LOGFILE) - { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file - if (iLogMode & MASTER) - { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file - } - } - continue; - } - case 'd': // signed decimal integer - case 'i': // signed decimal integer - { - // set iArg as the next value in list and print to console and/or files if flagged - iArg = va_arg(vaArgList, int); - if (iLogMode & LOGCONSOLE) - { - szNumWritten = printf(sFormatSpec.c_str(), iArg); // console - } - if (iLogMode & LOGFILE) - { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file - if (iLogMode & MASTER) - { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file - } - } - continue; - } - case 'u': // unsigned decimal integer - case 'o': // unsigned octal integer - case 'x': // unsigned hexadecimal integer using "abcdef" - case 'X': // unsigned hexadecimal integer using "ABCDEF" - { - // set uiArg as the next value in list and print to console and/or files if flagged - uiArg = va_arg(vaArgList, unsigned int); - if (iLogMode & LOGCONSOLE) - { - szNumWritten = printf(sFormatSpec.c_str(), uiArg); // console - } - if (iLogMode & LOGFILE) - { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file - if (iLogMode & MASTER) - { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file - } - } - continue; - } - case 'f': // float/double - case 'e': // scientific double/float - case 'E': // scientific double/float - case 'g': // scientific double/float - case 'G': // scientific double/float - case 'a': // signed hexadecimal double precision float - case 'A': // signed hexadecimal double precision float - { - // set dArg as the next value in list and print to console and/or files if flagged - dArg = va_arg(vaArgList, double); - if (iLogMode & LOGCONSOLE) - { - szNumWritten = printf(sFormatSpec.c_str(), dArg); // console - } - if (iLogMode & LOGFILE) - { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file - if (iLogMode & MASTER) - { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file - } - } - continue; - } - default: - { - // print arg of unknown/unsupported type to console and/or file if flagged - if (iLogMode & LOGCONSOLE) // console - { - szNumWritten = putc(*pStr, stdout); - } - if (iLogMode & LOGFILE) - { - szNumWritten = putc(*pStr, pFileStream0); // sample log file - if (iLogMode & MASTER) - { - szNumWritten = putc(*pStr, pFileStream1); // master log file - } - } - } - } - } - } - - // end the sample log with a horizontal line if closing - if (iLogMode & CLOSELOG) - { - if (iLogMode & LOGCONSOLE) - { - printf(HDASHLINE); - } - if (iLogMode & LOGFILE) - { - fprintf(pFileStream0, HDASHLINE); - } - } - - // flush console and/or file buffers if updated - if (iLogMode & LOGCONSOLE) - { - fflush(stdout); - } - if (iLogMode & LOGFILE) - { - fflush (pFileStream0); - - // if the master log file has been updated, flush it too - if (iLogMode & MASTER) - { - fflush (pFileStream1); - } - } - - // If the log file is open and the caller requests "close file", then close and NULL file handle - if ((pFileStream0) && (iLogMode & CLOSELOG)) - { - fclose (pFileStream0); - pFileStream0 = NULL; - } - if ((pFileStream1) && (iLogMode & CLOSELOG)) - { - fclose (pFileStream1); - pFileStream1 = NULL; - } - - // return error code or OK - if (iLogMode & ERRORMSG) - { - return iErrNum; - } - else - { - return 0; - } -} - -// Function to log standardized information to console, file or both -// ********************************************************************* -int shrLogEx(int iLogMode = LOGCONSOLE, int iErrNum = 0, const char* cFormatString = "", ...) -{ - va_list vaArgList; - - // Prepare variable agument list - va_start(vaArgList, cFormatString); - int ret = shrLogV(iLogMode, iErrNum, cFormatString, vaArgList); - - // end variable argument handler - va_end(vaArgList); - - return ret; -} - -// Function to log standardized information to console, file or both -// ********************************************************************* -int shrLog(const char* cFormatString = "", ...) -{ - va_list vaArgList; - - // Prepare variable agument list - va_start(vaArgList, cFormatString); - int ret = shrLogV(LOGBOTH, 0, cFormatString, vaArgList); - - // end variable argument handler - va_end(vaArgList); - - return ret; -} - -////////////////////////////////////////////////////////////////////////////// -//! Find the path for a file assuming that -//! files are found in the searchPath. -//! -//! @return the path if succeeded, otherwise 0 -//! @param filename name of the file -//! @param executable_path optional absolute path of the executable -////////////////////////////////////////////////////////////////////////////// -char* shrFindFilePath(const char* filename, const char* executable_path) -{ - // defines a variable that is replaced with the name of the executable - - // Typical relative search paths to locate needed companion files (e.g. sample input data, or JIT source files) - // The origin for the relative search may be the .exe file, a .bat file launching an .exe, a browser .exe launching the .exe or .bat, etc - const char* searchPath[] = - { - "./", // same dir - "./data/", // "/data/" subdir - "./src/", // "/src/" subdir - "./src//data/", // "/src//data/" subdir - "./inc/", // "/inc/" subdir - "../", // up 1 in tree - "../data/", // up 1 in tree, "/data/" subdir - "../src/", // up 1 in tree, "/src/" subdir - "../inc/", // up 1 in tree, "/inc/" subdir - "../OpenCL/src//", // up 1 in tree, "/OpenCL/src//" subdir - "../OpenCL/src//data/", // up 1 in tree, "/OpenCL/src//data/" subdir - "../OpenCL/src//src/", // up 1 in tree, "/OpenCL/src//src/" subdir - "../OpenCL/src//inc/", // up 1 in tree, "/OpenCL/src//inc/" subdir - "../C/src//", // up 1 in tree, "/C/src//" subdir - "../C/src//data/", // up 1 in tree, "/C/src//data/" subdir - "../C/src//src/", // up 1 in tree, "/C/src//src/" subdir - "../C/src//inc/", // up 1 in tree, "/C/src//inc/" subdir - "../DirectCompute/src//", // up 1 in tree, "/DirectCompute/src//" subdir - "../DirectCompute/src//data/", // up 1 in tree, "/DirectCompute/src//data/" subdir - "../DirectCompute/src//src/", // up 1 in tree, "/DirectCompute/src//src/" subdir - "../DirectCompute/src//inc/", // up 1 in tree, "/DirectCompute/src//inc/" subdir - "../../", // up 2 in tree - "../../data/", // up 2 in tree, "/data/" subdir - "../../src/", // up 2 in tree, "/src/" subdir - "../../inc/", // up 2 in tree, "/inc/" subdir - "../../../", // up 3 in tree - "../../../src//", // up 3 in tree, "/src//" subdir - "../../../src//data/", // up 3 in tree, "/src//data/" subdir - "../../../src//src/", // up 3 in tree, "/src//src/" subdir - "../../../src//inc/", // up 3 in tree, "/src//inc/" subdir - "../../../sandbox//", // up 3 in tree, "/sandbox//" subdir - "../../../sandbox//data/", // up 3 in tree, "/sandbox//data/" subdir - "../../../sandbox//src/", // up 3 in tree, "/sandbox//src/" subdir - "../../../sandbox//inc/" // up 3 in tree, "/sandbox//inc/" subdir - }; - - // Extract the executable name - std::string executable_name; - if (executable_path != 0) - { - executable_name = std::string(executable_path); - - #ifdef _WIN32 - // Windows path delimiter - size_t delimiter_pos = executable_name.find_last_of('\\'); - executable_name.erase(0, delimiter_pos + 1); - - if (executable_name.rfind(".exe") != string::npos) - { - // we strip .exe, only if the .exe is found - executable_name.resize(executable_name.size() - 4); - } - #else - // Linux & OSX path delimiter - size_t delimiter_pos = executable_name.find_last_of('/'); - executable_name.erase(0,delimiter_pos+1); - #endif - - } - - // Loop over all search paths and return the first hit - for( unsigned int i = 0; i < sizeof(searchPath)/sizeof(char*); ++i ) - { - std::string path(searchPath[i]); - size_t executable_name_pos = path.find(""); - - // If there is executable_name variable in the searchPath - // replace it with the value - if(executable_name_pos != std::string::npos) - { - if(executable_path != 0) - { - path.replace(executable_name_pos, strlen(""), executable_name); - - } - else - { - // Skip this path entry if no executable argument is given - continue; - } - } - - // Test if the file exists - path.append(filename); - std::fstream fh(path.c_str(), std::fstream::in); - if (fh.good()) - { - // File found - // returning an allocated array here for backwards compatibility reasons - char* file_path = (char*) malloc(path.length() + 1); - #ifdef _WIN32 - strcpy_s(file_path, path.length() + 1, path.c_str()); - #else - strcpy(file_path, path.c_str()); - #endif - return file_path; - } - } - - // File not found - return 0; -} - -////////////////////////////////////////////////////////////////////////////// -//! Read file \filename and return the data -//! @return shrTRUE if reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -////////////////////////////////////////////////////////////////////////////// -template -shrBOOL -shrReadFile( const char* filename, T** data, unsigned int* len, bool verbose) -{ - // check input arguments - ARGCHECK(NULL != filename); - ARGCHECK(NULL != len); - - // intermediate storage for the data read - std::vector data_read; - - // open file for reading - std::fstream fh( filename, std::fstream::in); - // check if filestream is valid - if(!fh.good()) - { - if (verbose) - std::cerr << "shrReadFile() : Opening file failed." << std::endl; - return shrFALSE; - } - - // read all data elements - T token; - while( fh.good()) - { - fh >> token; - data_read.push_back( token); - } - - // the last element is read twice - data_read.pop_back(); - - // check if reading result is consistent - if( ! fh.eof()) - { - if (verbose) - std::cerr << "WARNING : readData() : reading file might have failed." - << std::endl; - } - - fh.close(); - - // check if the given handle is already initialized - if( NULL != *data) - { - if( *len != data_read.size()) - { - std::cerr << "shrReadFile() : Initialized memory given but " - << "size mismatch with signal read " - << "(data read / data init = " << (unsigned int)data_read.size() - << " / " << *len << ")" << std::endl; - - return shrFALSE; - } - } - else - { - // allocate storage for the data read - *data = (T*) malloc( sizeof(T) * data_read.size()); - // store signal size - *len = static_cast( data_read.size()); - } - - // copy data - memcpy( *data, &data_read.front(), sizeof(T) * data_read.size()); - - return shrTRUE; -} - -////////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename -//! @return shrTRUE if writing the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data data to write -//! @param len number of data elements in data, -1 on error -//! @param epsilon epsilon for comparison -////////////////////////////////////////////////////////////////////////////// -template -shrBOOL -shrWriteFile( const char* filename, const T* data, unsigned int len, - const T epsilon, bool verbose) -{ - ARGCHECK(NULL != filename); - ARGCHECK(NULL != data); - - // open file for writing - std::fstream fh( filename, std::fstream::out); - // check if filestream is valid - if(!fh.good()) - { - if (verbose) - std::cerr << "shrWriteFile() : Opening file failed." << std::endl; - return shrFALSE; - } - - // first write epsilon - fh << "# " << epsilon << "\n"; - - // write data - for( unsigned int i = 0; (i < len) && (fh.good()); ++i) - { - fh << data[i] << ' '; - } - - // Check if writing succeeded - if( ! fh.good()) - { - if (verbose) - std::cerr << "shrWriteFile() : Writing file failed." << std::endl; - return shrFALSE; - } - - // file ends with nl - fh << std::endl; - - return shrTRUE; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Read file \filename containg single precision floating point data -//! @return shrTRUEif reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrReadFilef( const char* filename, float** data, unsigned int* len, bool verbose) -{ - return shrReadFile( filename, data, len, verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Read file \filename containg double precision floating point data -//! @return shrTRUEif reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrReadFiled( const char* filename, double** data, unsigned int* len, bool verbose) -{ - return shrReadFile( filename, data, len, verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Read file \filename containg integer data -//! @return shrTRUEif reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrReadFilei( const char* filename, int** data, unsigned int* len, bool verbose) -{ - return shrReadFile( filename, data, len, verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Read file \filename containg unsigned integer data -//! @return shrTRUEif reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrReadFileui( const char* filename, unsigned int** data, unsigned int* len, bool verbose) -{ - return shrReadFile( filename, data, len, verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Read file \filename containg char / byte data -//! @return shrTRUEif reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrReadFileb( const char* filename, char** data, unsigned int* len, bool verbose) -{ - return shrReadFile( filename, data, len, verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Read file \filename containg unsigned char / byte data -//! @return shrTRUEif reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrReadFileub( const char* filename, unsigned char** data, unsigned int* len, bool verbose) -{ - return shrReadFile( filename, data, len, verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename for single precision floating point data -//! @return shrTRUEif writing the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data data to write -//! @param len number of data elements in data, -1 on error -//! @param epsilon epsilon for comparison -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrWriteFilef( const char* filename, const float* data, unsigned int len, - const float epsilon, bool verbose) -{ - return shrWriteFile( filename, data, len, epsilon, verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename for double precision floating point data -//! @return shrTRUEif writing the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data data to write -//! @param len number of data elements in data, -1 on error -//! @param epsilon epsilon for comparison -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrWriteFiled( const char* filename, const double* data, unsigned int len, - const double epsilon, bool verbose) -{ - return shrWriteFile( filename, data, len, epsilon, verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename for integer data -//! @return shrTRUEif writing the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data data to write -//! @param len number of data elements in data, -1 on error -//! @param epsilon epsilon for comparison -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrWriteFilei( const char* filename, const int* data, unsigned int len, bool verbose) -{ - return shrWriteFile( filename, data, len, 0, verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename for unsigned integer data -//! @return shrTRUEif writing the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data data to write -//! @param len number of data elements in data, -1 on error -//! @param epsilon epsilon for comparison -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrWriteFileui( const char* filename,const unsigned int* data,unsigned int len, bool verbose) -{ - return shrWriteFile( filename, data, len, static_cast(0), verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename for byte / char data -//! @return shrTRUEif writing the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data data to write -//! @param len number of data elements in data, -1 on error -//! @param epsilon epsilon for comparison -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrWriteFileb( const char* filename, const char* data, unsigned int len, bool verbose) -{ - return shrWriteFile( filename, data, len, static_cast(0), verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename for byte / char data -//! @return shrTRUEif writing the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data data to write -//! @param len number of data elements in data, -1 on error -//! @param epsilon epsilon for comparison -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrWriteFileub( const char* filename, const unsigned char* data, - unsigned int len, bool verbose) -{ - return shrWriteFile( filename, data, len, static_cast(0), verbose); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename for unsigned byte / char data -//! @return shrTRUEif writing the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data data to write -//! @param len number of data elements in data, -1 on error -//! @param epsilon epsilon for comparison -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrWriteFileb( const char* filename,const unsigned char* data,unsigned int len, bool verbose) -{ - return shrWriteFile( filename, data, len, static_cast(0), verbose); -} - -////////////////////////////////////////////////////////////////////////////// -//! Load PGM or PPM file -//! @note if data == NULL then the necessary memory is allocated in the -//! function and w and h are initialized to the size of the image -//! @return shrTRUE if the file loading succeeded, otherwise shrFALSE -//! @param file name of the file to load -//! @param data handle to the memory for the image file data -//! @param w width of the image -//! @param h height of the image -//! @param channels number of channels in image -////////////////////////////////////////////////////////////////////////////// -shrBOOL loadPPM(const char* file, unsigned char** data, - unsigned int *w, unsigned int *h, unsigned int *channels) -{ - FILE* fp = 0; - - #ifdef _WIN32 - // open the file for binary read - errno_t err; - if ((err = fopen_s(&fp, file, "rb")) != 0) - #else - // open the file for binary read - if ((fp = fopen(file, "rb")) == 0) - #endif - { - // if error on attempt to open, be sure the file is null or close it, then return negative error code - if (fp) - { - fclose (fp); - } - std::cerr << "loadPPM() : Failed to open file: " << file << std::endl; - return shrFALSE; - } - - // check header - char header[PGMHeaderSize]; - if ((fgets( header, PGMHeaderSize, fp) == NULL) && ferror(fp)) - { - if (fp) - { - fclose (fp); - } - std::cerr << "loadPPM() : File is not a valid PPM or PGM image" << std::endl; - *channels = 0; - return shrFALSE; - } - - if (strncmp(header, "P5", 2) == 0) - { - *channels = 1; - } - else if (strncmp(header, "P6", 2) == 0) - { - *channels = 3; - } - else - { - std::cerr << "loadPPM() : File is not a PPM or PGM image" << std::endl; - *channels = 0; - return shrFALSE; - } - - // parse header, read maxval, width and height - unsigned int width = 0; - unsigned int height = 0; - unsigned int maxval = 0; - unsigned int i = 0; - while(i < 3) - { - if ((fgets(header, PGMHeaderSize, fp) == NULL) && ferror(fp)) - { - if (fp) - { - fclose (fp); - } - std::cerr << "loadPPM() : File is not a valid PPM or PGM image" << std::endl; - return shrFALSE; - } - if(header[0] == '#') continue; - - #ifdef _WIN32 - if(i == 0) - { - i += sscanf_s(header, "%u %u %u", &width, &height, &maxval); - } - else if (i == 1) - { - i += sscanf_s(header, "%u %u", &height, &maxval); - } - else if (i == 2) - { - i += sscanf_s(header, "%u", &maxval); - } - #else - if(i == 0) - { - i += sscanf(header, "%u %u %u", &width, &height, &maxval); - } - else if (i == 1) - { - i += sscanf(header, "%u %u", &height, &maxval); - } - else if (i == 2) - { - i += sscanf(header, "%u", &maxval); - } - #endif - } - - // check if given handle for the data is initialized - if(NULL != *data) - { - if (*w != width || *h != height) - { - fclose(fp); - std::cerr << "loadPPM() : Invalid image dimensions." << std::endl; - return shrFALSE; - } - } - else - { - *data = (unsigned char*)malloc( sizeof(unsigned char) * width * height * *channels); - *w = width; - *h = height; - } - - // read and close file - if (fread(*data, sizeof(unsigned char), width * height * *channels, fp) != width * height * *channels) - { - fclose(fp); - std::cerr << "loadPPM() : Invalid image." << std::endl; - return shrFALSE; - } - fclose(fp); - - return shrTRUE; -} - -////////////////////////////////////////////////////////////////////////////// -//! Write / Save PPM or PGM file -//! @note Internal usage only -//! @param file name of the image file -//! @param data handle to the data read -//! @param w width of the image -//! @param h height of the image -////////////////////////////////////////////////////////////////////////////// -shrBOOL savePPM( const char* file, unsigned char *data, - unsigned int w, unsigned int h, unsigned int channels) -{ - ARGCHECK(NULL != data); - ARGCHECK(w > 0); - ARGCHECK(h > 0); - - std::fstream fh( file, std::fstream::out | std::fstream::binary ); - if( fh.bad()) - { - std::cerr << "savePPM() : Opening file failed." << std::endl; - return shrFALSE; - } - - if (channels == 1) - { - fh << "P5\n"; - } - else if (channels == 3) { - fh << "P6\n"; - } - else { - std::cerr << "savePPM() : Invalid number of channels." << std::endl; - return shrFALSE; - } - - fh << w << "\n" << h << "\n" << 0xff << std::endl; - - for( unsigned int i = 0; (i < (w*h*channels)) && fh.good(); ++i) - { - fh << data[i]; - } - fh.flush(); - - if( fh.bad()) - { - std::cerr << "savePPM() : Writing data failed." << std::endl; - return shrFALSE; - } - fh.close(); - - return shrTRUE; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Load PPM image file (with unsigned char as data element type), padding 4th component -//! @return shrTrue if reading the file succeeded, otherwise shrFALSE -//! @param file name of the image file -//! @param data handle to the data read -//! @param w width of the image -//! @param h height of the image -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrLoadPPM4ub( const char* file, unsigned char** OutData, - unsigned int *w, unsigned int *h) -{ - // Load file data into a temporary buffer with automatic allocation - unsigned char* cLocalData = 0; - unsigned int channels; - shrBOOL bLoadOK = loadPPM(file, &cLocalData, w, h, &channels); // this allocates cLocalData, which must be freed later - - // If the data loaded OK from file to temporary buffer, then go ahead with padding and transfer - if (shrTRUE == bLoadOK) - { - // if the receiving buffer is null, allocate it... caller must free this - int size = *w * *h; - if (*OutData == NULL) - { - *OutData = (unsigned char*)malloc(sizeof(unsigned char) * size * 4); - } - - // temp pointers for incrementing - unsigned char* cTemp = cLocalData; - unsigned char* cOutPtr = *OutData; - - // transfer data, padding 4th element - for(int i=0; i( arg_name); - if( NULL != v) - { - // assign value - *val = *v; - ret_val = shrTRUE; - } - else { - // fail safe - val = NULL; - } - } - catch( const std::exception& /*ex*/) - { - std::cerr << "Error when parsing command line argument string." << std::endl; - } - - return ret_val; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Get the value of a command line argument of type unsigned int -//! @return shrTRUE if command line argument \a arg_name has been given and -//! is of the requested type, otherwise shrFALSE -//! @param argc argc as passed to main() -//! @param argv argv as passed to main() -//! @param arg_name name of the command line argument -//! @param val value of the command line argument -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrGetCmdLineArgumentu( const int argc, const char** argv, - const char* arg_name, unsigned int* val) -{ - shrBOOL ret_val = shrFALSE; - - try - { - // initialize - CmdArgReader::init( argc, argv); - - // access argument - const int* v = CmdArgReader::getArg( arg_name); - if( NULL != v) - { - // assign value - *val = *v; - ret_val = shrTRUE; - } - else { - // fail safe - val = NULL; - } - } - catch( const std::exception& /*ex*/) - { - std::cerr << "Error when parsing command line argument string." << std::endl; - } - - return ret_val; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Get the value of a command line argument of type float -//! @return shrTRUE if command line argument \a arg_name has been given and -//! is of the requested type, otherwise shrFALSE -//! @param argc argc as passed to main() -//! @param argv argv as passed to main() -//! @param arg_name name of the command line argument -//! @param val value of the command line argument -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrGetCmdLineArgumentf( const int argc, const char** argv, - const char* arg_name, float* val) -{ - shrBOOL ret_val = shrFALSE; - - try - { - // initialize - CmdArgReader::init( argc, argv); - - // access argument - const float* v = CmdArgReader::getArg( arg_name); - if( NULL != v) - { - // assign value - *val = *v; - ret_val = shrTRUE; - } - else { - // fail safe - val = NULL; - } - } - catch( const std::exception& /*ex*/) - { - std::cerr << "Error when parsing command line argument string." << std::endl; - } - - return ret_val; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Get the value of a command line argument of type string -//! @return shrTRUE if command line argument \a arg_name has been given and -//! is of the requested type, otherwise shrFALSE -//! @param argc argc as passed to main() -//! @param argv argv as passed to main() -//! @param arg_name name of the command line argument -//! @param val value of the command line argument -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrGetCmdLineArgumentstr( const int argc, const char** argv, - const char* arg_name, char** val) -{ - shrBOOL ret_val = shrFALSE; - - try - { - // initialize - CmdArgReader::init( argc, argv); - - // access argument - const std::string* v = CmdArgReader::getArg( arg_name); - if( NULL != v) - { - - // allocate memory for the string - *val = (char*)malloc(sizeof(char) * (v->length() + 1)); - - // copy from string to c_str - #ifdef WIN32 - strcpy_s(*val, v->length() + 1, v->c_str()); - #else - strcpy(*val, v->c_str()); - #endif - ret_val = shrTRUE; - } - else { - // fail safe - *val = NULL; - } - } - catch( const std::exception& /*ex*/) - { - std::cerr << "Error when parsing command line argument string."<< - std::endl; - } - - return ret_val; - -} - -////////////////////////////////////////////////////////////////////////////// -//! Compare two arrays of arbitrary type -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param epsilon epsilon to use for the comparison -////////////////////////////////////////////////////////////////////////////// -template -shrBOOL -compareData( const T* reference, const T* data, const unsigned int len, - const S epsilon, const float threshold) -{ - ARGCHECK( epsilon >= 0); - - bool result = true; - unsigned int error_count = 0; - - for( unsigned int i = 0; i < len; ++i) { - - T diff = reference[i] - data[i]; - bool comp = (diff <= epsilon) && (diff >= -epsilon); - result &= comp; - - error_count += !comp; - -#ifdef _DEBUG - if( ! comp) - { - std::cerr << "ERROR, i = " << i << ",\t " - << reference[i] << " / " - << data[i] - << " (reference / data)\n"; - } -#endif - } - - if (threshold == 0.0f) { - return (result) ? shrTRUE : shrFALSE; - } else { - return (len*threshold > error_count) ? shrTRUE : shrFALSE; - } -} - -////////////////////////////////////////////////////////////////////////////// -//! Compare two arrays of arbitrary type -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param epsilon epsilon to use for the comparison -////////////////////////////////////////////////////////////////////////////// -template -shrBOOL -compareDataAsFloat( const T* reference, const T* data, const unsigned int len, - const S epsilon) -{ - ARGCHECK(epsilon >= 0); - - // If we set epsilon to be 0, let's set a minimum threshold - float max_error = MAX( (float)epsilon, MIN_EPSILON_ERROR ); - int error_count = 0; - bool result = true; - - for( unsigned int i = 0; i < len; ++i) { - float diff = fabs((float)reference[i] - (float)data[i]); - bool comp = (diff < max_error); - result &= comp; - - if( ! comp) - { - error_count++; -#ifdef _DEBUG - if (error_count < 50) { - shrLog("\n ERROR(epsilon=%4.3f), i=%d, (ref)0x%02x / (data)0x%02x / (diff)%d\n", max_error, i, reference[i], data[i], (unsigned int)diff); - } -#endif - } - } - if (error_count) { - shrLog("\n Total # of errors = %d\n", error_count); - } - return (error_count == 0) ? shrTRUE : shrFALSE; -} - -////////////////////////////////////////////////////////////////////////////// -//! Compare two arrays of arbitrary type -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param epsilon epsilon to use for the comparison -//! @param epsilon threshold % of (# of bytes) for pass/fail -////////////////////////////////////////////////////////////////////////////// -template -shrBOOL -compareDataAsFloatThreshold( const T* reference, const T* data, const unsigned int len, - const S epsilon, const float threshold) -{ - ARGCHECK(epsilon >= 0); - - // If we set epsilon to be 0, let's set a minimum threshold - float max_error = MAX( (float)epsilon, MIN_EPSILON_ERROR); - int error_count = 0; - bool result = true; - - for( unsigned int i = 0; i < len; ++i) { - float diff = fabs((float)reference[i] - (float)data[i]); - bool comp = (diff < max_error); - result &= comp; - - if( ! comp) - { - error_count++; -//#ifdef _DEBUG - if (error_count < 50) { - shrLog("\n ERROR(epsilon=%4.3f), i=%d, (ref)%f / (data)%f / (diff)%f\n", max_error, i, reference[i], data[i], diff); - } -//#endif - } - } - - if (threshold == 0.0f) { - if (error_count) { - shrLog("\n Total # of errors = %d\n", error_count); - } - return (error_count == 0) ? shrTRUE : shrFALSE; - } else { - - if (error_count) { - shrLog("\n %.2f(%%) of bytes mismatched (count=%d)\n", (float)error_count*100/(float)len, error_count); - } - - return ((len*threshold > error_count) ? shrTRUE : shrFALSE); - } -} - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two float arrays -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrComparef( const float* reference, const float* data, - const unsigned int len ) -{ - const float epsilon = 0.0; - return compareData( reference, data, len, epsilon, 0.0f ); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two integer arrays -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrComparei( const int* reference, const int* data, - const unsigned int len ) -{ - const int epsilon = 0; - return compareData( reference, data, len, epsilon, 0.0f); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two unsigned integer arrays, with epsilon and threshold -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrCompareuit( const unsigned int* reference, const unsigned int* data, - const unsigned int len, const float epsilon, const float threshold ) -{ - return compareDataAsFloatThreshold( reference, data, len, epsilon, threshold ); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two integer arrays -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrCompareub( const unsigned char* reference, const unsigned char* data, - const unsigned int len ) -{ - const int epsilon = 0; - return compareData( reference, data, len, epsilon, 0.0f); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two integer arrays (inc Threshold for # of pixel we can have errors) -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrCompareubt( const unsigned char* reference, const unsigned char* data, - const unsigned int len, const float epsilon, const float threshold ) -{ - return compareDataAsFloatThreshold( reference, data, len, epsilon, threshold ); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two integer arrays -//! @return shrTRUE if \a reference and \a data are identical, -//! otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrCompareube( const unsigned char* reference, const unsigned char* data, - const unsigned int len, const float epsilon ) -{ - return compareDataAsFloat( reference, data, len, epsilon ); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two float arrays with an epsilon tolerance for equality -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param epsilon epsilon to use for the comparison -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrComparefe( const float* reference, const float* data, - const unsigned int len, const float epsilon ) -{ - return compareData( reference, data, len, epsilon, 0.0f); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two float arrays with an epsilon tolerance for equality and a -//! threshold for # pixel errors -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param epsilon epsilon to use for the comparison -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrComparefet( const float* reference, const float* data, - const unsigned int len, const float epsilon, const float threshold ) -{ - return compareDataAsFloatThreshold( reference, data, len, epsilon, threshold ); -} - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two float arrays using L2-norm with an epsilon tolerance for equality -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param epsilon epsilon to use for the comparison -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrCompareL2fe( const float* reference, const float* data, - const unsigned int len, const float epsilon ) -{ - ARGCHECK(epsilon >= 0); - - float error = 0; - float ref = 0; - - for( unsigned int i = 0; i < len; ++i) { - - float diff = reference[i] - data[i]; - error += diff * diff; - ref += reference[i] * reference[i]; - } - - float normRef = sqrtf(ref); - if (fabs(ref) < 1e-7) { -#ifdef _DEBUG - std::cerr << "ERROR, reference l2-norm is 0\n"; -#endif - return shrFALSE; - } - float normError = sqrtf(error); - error = normError / normRef; - bool result = error < epsilon; -#ifdef _DEBUG - if( ! result) - { - std::cerr << "ERROR, l2-norm error " - << error << " is greater than epsilon " << epsilon << "\n"; - } -#endif - - return result ? shrTRUE : shrFALSE; -} - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two PPM image files with an epsilon tolerance for equality -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param src_file filename for the image to be compared -//! @param data filename for the reference data / gold image -//! @param epsilon epsilon to use for the comparison -//! @param threshold threshold of pixels that can still mismatch to pass (i.e. 0.15f = 15% must pass) -//! @param verboseErrors output details of image mismatch to std::cerr -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrComparePPM( const char *src_file, const char *ref_file, const float epsilon, const float threshold) -{ - unsigned char* src_data = NULL; - unsigned char* ref_data = NULL; - unsigned long error_count = 0; - unsigned int ref_width, ref_height; - unsigned int src_width, src_height; - - // Check sample and reference file pointers - if (src_file == NULL || ref_file == NULL) { - shrLog("\n> shrComparePGM: src_file or ref_file is NULL\n Aborting comparison !!!\n\n"); - return shrFALSE; - } - shrLog("\n> shrComparePPM:\n (a)rendered: <%s>\n (b)reference: <%s>\n", src_file, ref_file); - - // Load the ref image file - if (shrLoadPPM4ub(ref_file, &ref_data, &ref_width, &ref_height) != shrTRUE) - { - shrLog("\n Unable to load ref image file: %s\n Aborting comparison !!!\n\n", ref_file); - return shrFALSE; - } - - // Load the sample image file - if (shrLoadPPM4ub(src_file, &src_data, &src_width, &src_height) != shrTRUE) - { - shrLog("\n Unable to load src image file: %s\n Aborting comparison !!!\n\n", src_file); - return shrFALSE; - } - - // check to see if image dimensions match - if(src_height != ref_height || src_width != ref_width) - { - shrLog("\n Source and ref size mismatch (%u x %u) vs (%u x %u)\n Aborting Comparison !!!\n\n ", - src_width, src_height, ref_width, ref_height); - return shrFALSE; - } - - // compare the images - if (shrCompareubt(ref_data, src_data, src_width*src_height*4, epsilon, threshold ) == shrFALSE) - { - error_count=1; - } - - shrLog(" Images %s\n\n", (error_count == 0) ? "Match" : "Don't Match !!!"); - return (error_count == 0) ? shrTRUE : shrFALSE; // returns true if all pixels pass -} - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two PGM image files with an epsilon tolerance for equality -//! @return shrTRUE if \a reference and \a data are identical, otherwise shrFALSE -//! @param src_file filename for the image to be compared -//! @param data filename for the reference data / gold image -//! @param epsilon epsilon to use for the comparison -//! @param threshold threshold of pixels that can still mismatch to pass (i.e. 0.15f = 15% must pass) -//////////////////////////////////////////////////////////////////////////////// -shrBOOL shrComparePGM( const char *src_file, const char *ref_file, const float epsilon, const float threshold) -{ - unsigned char* src_data = NULL; - unsigned char* ref_data = NULL; - unsigned long error_count = 0; - unsigned int ref_width, ref_height; - unsigned int src_width, src_height; - - // Check sample and reference file pointers - if (src_file == NULL || ref_file == NULL) { - shrLog("\n> shrComparePGM: src_file or ref_file is NULL\n Aborting comparison !!!\n\n"); - return shrFALSE; - } - shrLog("\n> shrComparePGM:\n (a)rendered: <%s>\n (b)reference: <%s>\n", src_file, ref_file); - - // Load the ref image file - if (shrLoadPPM4ub(ref_file, &ref_data, &ref_width, &ref_height) != shrTRUE) - { - shrLog("\n Unable to load ref image file: %s\n Aborting comparison !!!\n\n", ref_file); - return shrFALSE; - } - - // Load the sample image file - if (shrLoadPPM4ub(src_file, &src_data, &src_width, &src_height) != shrTRUE) - { - shrLog("\n Unable to load src image file: %s\n Aborting comparison !!!\n\n", src_file); - return shrFALSE; - } - - // check to see if image dimensions match - if(src_height != ref_height || src_width != ref_width) - { - shrLog("\n Source and ref size mismatch (%u x %u) vs (%u x %u)\n Aborting Comparison !!!\n\n ", - src_width, src_height, ref_width, ref_height); - return shrFALSE; - } - - // compare the images - if (shrCompareubt(ref_data, src_data, src_width*src_height*4, epsilon, threshold ) == shrFALSE) - { - error_count=1; - } - - shrLog(" Images %s\n\n", (error_count == 0) ? "Match" : "Don't Match !!!"); - return (error_count == 0) ? shrTRUE : shrFALSE; // returns true if all pixels pass -} - -// Load raw data from disk -unsigned char* shrLoadRawFile(const char* filename, size_t size) -{ - FILE *fp = NULL; - #ifdef WIN32 - errno_t err; - if ((err = fopen_s(&fp, filename, "rb")) != 0) - #else - if ((fp = fopen(filename, "rb")) == NULL) - #endif - { - shrLog(" Error opening file '%s' !!!\n", filename); - return 0; - } - - unsigned char* data = (unsigned char*)malloc(size); - size_t read = fread(data, 1, size, fp); - fclose(fp); - - shrLog(" Read '%s', %d bytes\n", filename, read); - - return data; -} - -// Round Up Division function -size_t shrRoundUp(int group_size, int global_size) -{ - int r = global_size % group_size; - if(r == 0) - { - return global_size; - } else - { - return global_size + group_size - r; - } -} \ No newline at end of file diff --git a/tests/opencl/vectorhypot/shrUtils.h b/tests/opencl/vectorhypot/shrUtils.h deleted file mode 100644 index 0f2795d4..00000000 --- a/tests/opencl/vectorhypot/shrUtils.h +++ /dev/null @@ -1,642 +0,0 @@ -/* -* Copyright 1993-2010 NVIDIA Corporation. All rights reserved. -* -* Please refer to the NVIDIA end user license agreement (EULA) associated -* with this source code for terms and conditions that govern your use of -* this software. Any use, reproduction, disclosure, or distribution of -* this software and related documentation outside the terms of the EULA -* is strictly prohibited. -* -*/ - -#ifndef SHR_UTILS_H -#define SHR_UTILS_H - -// ********************************************************************* -// Generic utilities for NVIDIA GPU Computing SDK -// ********************************************************************* - -// reminders for output window and build log -#ifdef _WIN32 - #pragma message ("Note: including windows.h") - #pragma message ("Note: including math.h") - #pragma message ("Note: including assert.h") -#endif - -// OS dependent includes -#ifdef _WIN32 - // Headers needed for Windows - #include -#else - // Headers needed for Linux - #include - #include - #include - #include - #include - #include - #include -#endif - -// Other headers needed for both Windows and Linux -#include -#include -#include -#include -#include - -// Un-comment the following #define to enable profiling code in SDK apps -//#define GPU_PROFILING - -// Beginning of GPU Architecture definitions -inline int ConvertSMVer2Cores(int major, int minor) -{ - // Defines for GPU Architecture types (using the SM version to determine the # of cores per SM - typedef struct { - int SM; // 0xMm (hexidecimal notation), M = SM Major version, and m = SM minor version - int Cores; - } sSMtoCores; - - sSMtoCores nGpuArchCoresPerSM[] = - { { 0x10, 8 }, // Tesla Generation (SM 1.0) G80 class - { 0x11, 8 }, // Tesla Generation (SM 1.1) G8x class - { 0x12, 8 }, // Tesla Generation (SM 1.2) G9x class - { 0x13, 8 }, // Tesla Generation (SM 1.3) GT200 class - { 0x20, 32 }, // Fermi Generation (SM 2.0) GF100 class - { 0x21, 48 }, // Fermi Generation (SM 2.1) GF10x class - { 0x30, 192}, // Fermi Generation (SM 3.0) GK10x class - { -1, -1 } - }; - - int index = 0; - while (nGpuArchCoresPerSM[index].SM != -1) { - if (nGpuArchCoresPerSM[index].SM == ((major << 4) + minor) ) { - return nGpuArchCoresPerSM[index].Cores; - } - index++; - } - printf("MapSMtoCores SM %d.%d is undefined (please update to the latest SDK)!\n", major, minor); - return -1; -} -// end of GPU Architecture definitions - - -// Defines and enum for use with logging functions -// ********************************************************************* -#define DEFAULTLOGFILE "SdkConsoleLog.txt" -#define MASTERLOGFILE "SdkMasterLog.csv" -enum LOGMODES -{ - LOGCONSOLE = 1, // bit to signal "log to console" - LOGFILE = 2, // bit to signal "log to file" - LOGBOTH = 3, // convenience union of first 2 bits to signal "log to both" - APPENDMODE = 4, // bit to set "file append" mode instead of "replace mode" on open - MASTER = 8, // bit to signal master .csv log output - ERRORMSG = 16, // bit to signal "pre-pend Error" - CLOSELOG = 32 // bit to close log file, if open, after any requested file write -}; -#define HDASHLINE "-----------------------------------------------------------\n" - -// Standardized boolean -enum shrBOOL -{ - shrFALSE = 0, - shrTRUE = 1 -}; - -// Standardized MAX, MIN and CLAMP -#define MAX(a, b) ((a > b) ? a : b) -#define MIN(a, b) ((a < b) ? a : b) -#define CLAMP(a, b, c) MIN(MAX(a, b), c) // double sided clip of input a -#define TOPCLAMP(a, b) (a < b ? a:b) // single top side clip of input a - -// Error and Exit Handling Macros... -// ********************************************************************* -// Full error handling macro with Cleanup() callback (if supplied)... -// (Companion Inline Function lower on page) -#define shrCheckErrorEX(a, b, c) __shrCheckErrorEX(a, b, c, __FILE__ , __LINE__) - -// Short version without Cleanup() callback pointer -// Both Input (a) and Reference (b) are specified as args -#define shrCheckError(a, b) shrCheckErrorEX(a, b, 0) - -// Standardized Exit Macro for leaving main()... extended version -// (Companion Inline Function lower on page) -#define shrExitEX(a, b, c) __shrExitEX(a, b, c) - -// Standardized Exit Macro for leaving main()... short version -// (Companion Inline Function lower on page) -#define shrEXIT(a, b) __shrExitEX(a, b, EXIT_SUCCESS) - -// Simple argument checker macro -#define ARGCHECK(a) if((a) != shrTRUE)return shrFALSE - -// Define for user-customized error handling -#define STDERROR "file %s, line %i\n\n" , __FILE__ , __LINE__ - -// Function to deallocate memory allocated within shrUtils -// ********************************************************************* -extern "C" void shrFree(void* ptr); - -// ********************************************************************* -// Helper function to log standardized information to Console, to File or to both -//! Examples: shrLogEx(LOGBOTH, 0, "Function A\n"); -//! : shrLogEx(LOGBOTH | ERRORMSG, ciErrNum, STDERROR); -//! -//! Automatically opens file and stores handle if needed and not done yet -//! Closes file and nulls handle on request -//! -//! @param 0 iLogMode: LOGCONSOLE, LOGFILE, LOGBOTH, APPENDMODE, MASTER, ERRORMSG, CLOSELOG. -//! LOGFILE and LOGBOTH may be | 'd with APPENDMODE to select file append mode instead of overwrite mode -//! LOGFILE and LOGBOTH may be | 'd with CLOSELOG to "write and close" -//! First 3 options may be | 'd with MASTER to enable independent write to master data log file -//! First 3 options may be | 'd with ERRORMSG to start line with standard error message -//! @param 2 dValue: -//! Positive val = double value for time in secs to be formatted to 6 decimals. -//! Negative val is an error code and this give error preformatting. -//! @param 3 cFormatString: String with formatting specifiers like printf or fprintf. -//! ALL printf flags, width, precision and type specifiers are supported with this exception: -//! Wide char type specifiers intended for wprintf (%S and %C) are NOT supported -//! Single byte char type specifiers (%s and %c) ARE supported -//! @param 4... variable args: like printf or fprintf. Must match format specifer type above. -//! @return 0 if OK, negative value on error or if error occurs or was passed in. -// ********************************************************************* -extern "C" int shrLogEx(int iLogMode, int iErrNum, const char* cFormatString, ...); - -// Short version of shrLogEx defaulting to shrLogEx(LOGBOTH, 0, -// ********************************************************************* -extern "C" int shrLog(const char* cFormatString, ...); - -// ********************************************************************* -// Delta timer function for up to 3 independent timers using host high performance counters -// Maintains state for 3 independent counters -//! Example: double dElapsedTime = shrDeltaTime(0); -//! -//! @param 0 iCounterID: Which timer to check/reset. (0, 1, 2) -//! @return delta time of specified counter since last call in seconds. Otherwise -9999.0 if error -// ********************************************************************* -extern "C" double shrDeltaT(int iCounterID); - -// Optional LogFileNameOverride function -// ********************************************************************* -extern "C" void shrSetLogFileName (const char* cOverRideName); - -// Helper function to init data arrays -// ********************************************************************* -extern "C" void shrFillArray(float* pfData, int iSize); - -// Helper function to print data arrays -// ********************************************************************* -extern "C" void shrPrintArray(float* pfData, int iSize); - -//////////////////////////////////////////////////////////////////////////// -//! Find the path for a filename -//! @return the path if succeeded, otherwise 0 -//! @param filename name of the file -//! @param executablePath optional absolute path of the executable -//////////////////////////////////////////////////////////////////////////// -extern "C" char* shrFindFilePath(const char* filename, const char* executablePath); - -//////////////////////////////////////////////////////////////////////////// -//! Read file \filename containing single precision floating point data -//! @return shrTRUE if reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//! @note If a NULL pointer is passed to this function and it is initialized -//! within shrUtils, then free() has to be used to deallocate the memory -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrReadFilef( const char* filename, float** data, unsigned int* len, - bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Read file \filename containing double precision floating point data -//! @return shrTRUE if reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//! @note If a NULL pointer is passed to this function and it is -//! @note If a NULL pointer is passed to this function and it is initialized -//! within shrUtils, then free() has to be used to deallocate the memory -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrReadFiled( const char* filename, double** data, unsigned int* len, - bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Read file \filename containing integer data -//! @return shrTRUE if reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//! @note If a NULL pointer is passed to this function and it is -//! @note If a NULL pointer is passed to this function and it is initialized -//! within shrUtils, then free() has to be used to deallocate the memory -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrReadFilei( const char* filename, int** data, unsigned int* len, bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Read file \filename containing unsigned integer data -//! @return shrTRUE if reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//! @note If a NULL pointer is passed to this function and it is -//! @note If a NULL pointer is passed to this function and it is initialized -//! within shrUtils, then free() has to be used to deallocate the memory -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrReadFileui( const char* filename, unsigned int** data, - unsigned int* len, bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Read file \filename containing char / byte data -//! @return shrTRUE if reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//! @note If a NULL pointer is passed to this function and it is -//! @note If a NULL pointer is passed to this function and it is initialized -//! within shrUtils, then free() has to be used to deallocate the memory -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrReadFileb( const char* filename, char** data, unsigned int* len, - bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Read file \filename containing unsigned char / byte data -//! @return shrTRUE if reading the file succeeded, otherwise shrFALSE -//! @param filename name of the source file -//! @param data uninitialized pointer, returned initialized and pointing to -//! the data read -//! @param len number of data elements in data, -1 on error -//! @note If a NULL pointer is passed to this function and it is -//! @note If a NULL pointer is passed to this function and it is initialized -//! within shrUtils, then free() has to be used to deallocate the memory -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrReadFileub( const char* filename, unsigned char** data, - unsigned int* len, bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename containing single precision floating point -//! data -//! @return shrTRUE if writing the file succeeded, otherwise shrFALSE -//! @param filename name of the file to write -//! @param data pointer to data to write -//! @param len number of data elements in data, -1 on error -//! @param epsilon epsilon for comparison -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrWriteFilef( const char* filename, const float* data, unsigned int len, - const float epsilon, bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename containing double precision floating point -//! data -//! @return shrTRUE if writing the file succeeded, otherwise shrFALSE -//! @param filename name of the file to write -//! @param data pointer to data to write -//! @param len number of data elements in data, -1 on error -//! @param epsilon epsilon for comparison -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrWriteFiled( const char* filename, const float* data, unsigned int len, - const double epsilon, bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename containing integer data -//! @return shrTRUE if writing the file succeeded, otherwise shrFALSE -//! @param filename name of the file to write -//! @param data pointer to data to write -//! @param len number of data elements in data, -1 on error -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrWriteFilei( const char* filename, const int* data, unsigned int len, - bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename containing unsigned integer data -//! @return shrTRUE if writing the file succeeded, otherwise shrFALSE -//! @param filename name of the file to write -//! @param data pointer to data to write -//! @param len number of data elements in data, -1 on error -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrWriteFileui( const char* filename, const unsigned int* data, - unsigned int len, bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename containing char / byte data -//! @return shrTRUE if writing the file succeeded, otherwise shrFALSE -//! @param filename name of the file to write -//! @param data pointer to data to write -//! @param len number of data elements in data, -1 on error -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrWriteFileb( const char* filename, const char* data, unsigned int len, - bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Write a data file \filename containing unsigned char / byte data -//! @return shrTRUE if writing the file succeeded, otherwise shrFALSE -//! @param filename name of the file to write -//! @param data pointer to data to write -//! @param len number of data elements in data, -1 on error -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrWriteFileub( const char* filename, const unsigned char* data, - unsigned int len, bool verbose = false); - -//////////////////////////////////////////////////////////////////////////// -//! Load PPM image file (with unsigned char as data element type), padding -//! 4th component -//! @return shrTRUE if reading the file succeeded, otherwise shrFALSE -//! @param file name of the image file -//! @param OutData handle to the data read -//! @param w width of the image -//! @param h height of the image -//! -//! Note: If *OutData is NULL this function allocates buffer that must be freed by caller -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrLoadPPM4ub(const char* file, unsigned char** OutData, - unsigned int *w, unsigned int *h); - -//////////////////////////////////////////////////////////////////////////// -//! Save PPM image file (with unsigned char as data element type, padded to -//! 4 bytes) -//! @return shrTRUE if saving the file succeeded, otherwise shrFALSE -//! @param file name of the image file -//! @param data handle to the data read -//! @param w width of the image -//! @param h height of the image -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrSavePPM4ub( const char* file, unsigned char *data, - unsigned int w, unsigned int h); - -//////////////////////////////////////////////////////////////////////////////// -//! Save PGM image file (with unsigned char as data element type) -//! @return shrTRUE if saving the file succeeded, otherwise shrFALSE -//! @param file name of the image file -//! @param data handle to the data read -//! @param w width of the image -//! @param h height of the image -//////////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrSavePGMub( const char* file, unsigned char *data, - unsigned int w, unsigned int h); - -//////////////////////////////////////////////////////////////////////////// -//! Load PGM image file (with unsigned char as data element type) -//! @return shrTRUE if saving the file succeeded, otherwise shrFALSE -//! @param file name of the image file -//! @param data handle to the data read -//! @param w width of the image -//! @param h height of the image -//! @note If a NULL pointer is passed to this function and it is initialized -//! within shrUtils, then free() has to be used to deallocate the memory -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrLoadPGMub( const char* file, unsigned char** data, - unsigned int *w,unsigned int *h); - -//////////////////////////////////////////////////////////////////////////// -// Command line arguments: General notes -// * All command line arguments begin with '--' followed by the token; -// token and value are seperated by '='; example --samples=50 -// * Arrays have the form --model=[one.obj,two.obj,three.obj] -// (without whitespaces) -//////////////////////////////////////////////////////////////////////////// - -//////////////////////////////////////////////////////////////////////////// -//! Check if command line argument \a flag-name is given -//! @return shrTRUE if command line argument \a flag_name has been given, -//! otherwise shrFALSE -//! @param argc argc as passed to main() -//! @param argv argv as passed to main() -//! @param flag_name name of command line flag -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrCheckCmdLineFlag( const int argc, const char** argv, - const char* flag_name); - -//////////////////////////////////////////////////////////////////////////// -//! Get the value of a command line argument of type int -//! @return shrTRUE if command line argument \a arg_name has been given and -//! is of the requested type, otherwise shrFALSE -//! @param argc argc as passed to main() -//! @param argv argv as passed to main() -//! @param arg_name name of the command line argument -//! @param val value of the command line argument -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrGetCmdLineArgumenti( const int argc, const char** argv, - const char* arg_name, int* val); - -//////////////////////////////////////////////////////////////////////////// -//! Get the value of a command line argument of type unsigned int -//! @return shrTRUE if command line argument \a arg_name has been given and -//! is of the requested type, otherwise shrFALSE -//! @param argc argc as passed to main() -//! @param argv argv as passed to main() -//! @param arg_name name of the command line argument -//! @param val value of the command line argument -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrGetCmdLineArgumentu( const int argc, const char** argv, - const char* arg_name, unsigned int* val); - -//////////////////////////////////////////////////////////////////////////// -//! Get the value of a command line argument of type float -//! @return shrTRUE if command line argument \a arg_name has been given and -//! is of the requested type, otherwise shrFALSE -//! @param argc argc as passed to main() -//! @param argv argv as passed to main() -//! @param arg_name name of the command line argument -//! @param val value of the command line argument -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrGetCmdLineArgumentf( const int argc, const char** argv, - const char* arg_name, float* val); - -//////////////////////////////////////////////////////////////////////////// -//! Get the value of a command line argument of type string -//! @return shrTRUE if command line argument \a arg_name has been given and -//! is of the requested type, otherwise shrFALSE -//! @param argc argc as passed to main() -//! @param argv argv as passed to main() -//! @param arg_name name of the command line argument -//! @param val value of the command line argument -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrGetCmdLineArgumentstr( const int argc, const char** argv, - const char* arg_name, char** val); - -//////////////////////////////////////////////////////////////////////////// -//! Get the value of a command line argument list those element are strings -//! @return shrTRUE if command line argument \a arg_name has been given and -//! is of the requested type, otherwise shrFALSE -//! @param argc argc as passed to main() -//! @param argv argv as passed to main() -//! @param arg_name name of the command line argument -//! @param val command line argument list -//! @param len length of the list / number of elements -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrGetCmdLineArgumentListstr( const int argc, const char** argv, - const char* arg_name, char** val, - unsigned int* len); - -//////////////////////////////////////////////////////////////////////////// -//! Compare two float arrays -//! @return shrTRUEif \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrComparef( const float* reference, const float* data, - const unsigned int len); - -//////////////////////////////////////////////////////////////////////////// -//! Compare two integer arrays -//! @return shrTRUEif \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrComparei( const int* reference, const int* data, - const unsigned int len ); - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two unsigned integer arrays, with epsilon and threshold -//! @return shrTRUEif \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param threshold tolerance % # of comparison errors (0.15f = 15%) -//////////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrCompareuit( const unsigned int* reference, const unsigned int* data, - const unsigned int len, const float epsilon, const float threshold ); - -//////////////////////////////////////////////////////////////////////////// -//! Compare two unsigned char arrays -//! @return shrTRUEif \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrCompareub( const unsigned char* reference, const unsigned char* data, - const unsigned int len ); - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two integers with a tolernance for # of byte errors -//! @return shrTRUEif \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param epsilon epsilon to use for the comparison -//! @param threshold tolerance % # of comparison errors (0.15f = 15%) -//////////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrCompareubt( const unsigned char* reference, const unsigned char* data, - const unsigned int len, const float epsilon, const float threshold ); - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two integer arrays witha n epsilon tolerance for equality -//! @return shrTRUEif \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param epsilon epsilon to use for the comparison -//////////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrCompareube( const unsigned char* reference, const unsigned char* data, - const unsigned int len, const float epsilon ); - -//////////////////////////////////////////////////////////////////////////// -//! Compare two float arrays with an epsilon tolerance for equality -//! @return shrTRUEif \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param epsilon epsilon to use for the comparison -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrComparefe( const float* reference, const float* data, - const unsigned int len, const float epsilon ); - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two float arrays with an epsilon tolerance for equality and a -//! threshold for # pixel errors -//! @return shrTRUEif \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param epsilon epsilon to use for the comparison -//////////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrComparefet( const float* reference, const float* data, - const unsigned int len, const float epsilon, const float threshold ); - -//////////////////////////////////////////////////////////////////////////// -//! Compare two float arrays using L2-norm with an epsilon tolerance for -//! equality -//! @return shrTRUEif \a reference and \a data are identical, otherwise shrFALSE -//! @param reference handle to the reference data / gold image -//! @param data handle to the computed data -//! @param len number of elements in reference and data -//! @param epsilon epsilon to use for the comparison -//////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrCompareL2fe( const float* reference, const float* data, - const unsigned int len, const float epsilon ); - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two PPM image files with an epsilon tolerance for equality -//! @return shrTRUEif \a reference and \a data are identical, otherwise shrFALSE -//! @param src_file filename for the image to be compared -//! @param data filename for the reference data / gold image -//! @param epsilon epsilon to use for the comparison -//! @param threshold threshold of pixels that can still mismatch to pass (i.e. 0.15f = 15% must pass) -//! $param verboseErrors output details of image mismatch to std::err -//////////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrComparePPM( const char *src_file, const char *ref_file, const float epsilon, const float threshold); - -//////////////////////////////////////////////////////////////////////////////// -//! Compare two PGM image files with an epsilon tolerance for equality -//! @return shrTRUEif \a reference and \a data are identical, otherwise shrFALSE -//! @param src_file filename for the image to be compared -//! @param data filename for the reference data / gold image -//! @param epsilon epsilon to use for the comparison -//! @param threshold threshold of pixels that can still mismatch to pass (i.e. 0.15f = 15% must pass) -//! $param verboseErrors output details of image mismatch to std::err -//////////////////////////////////////////////////////////////////////////////// -extern "C" shrBOOL shrComparePGM( const char *src_file, const char *ref_file, const float epsilon, const float threshold); - -extern "C" unsigned char* shrLoadRawFile(const char* filename, size_t size); - -extern "C" size_t shrRoundUp(int group_size, int global_size); - -// companion inline function for error checking and exit on error WITH Cleanup Callback (if supplied) -// ********************************************************************* -inline void __shrCheckErrorEX(int iSample, int iReference, void (*pCleanup)(int), const char* cFile, const int iLine) -{ - if (iReference != iSample) - { - shrLogEx(LOGBOTH | ERRORMSG, iSample, "line %i , in file %s !!!\n\n" , iLine, cFile); - if (pCleanup != NULL) - { - pCleanup(EXIT_FAILURE); - } - else - { - shrLogEx(LOGBOTH | CLOSELOG, 0, "Exiting...\n"); - exit(EXIT_FAILURE); - } - } -} - -// Standardized Exit -// ********************************************************************* -inline void __shrExitEX(int argc, const char** argv, int iExitCode) -{ -#ifdef WIN32 - if (!shrCheckCmdLineFlag(argc, argv, "noprompt") && !shrCheckCmdLineFlag(argc, argv, "qatest")) -#else - if (shrCheckCmdLineFlag(argc, argv, "prompt") && !shrCheckCmdLineFlag(argc, argv, "qatest")) -#endif - { - shrLogEx(LOGBOTH | CLOSELOG, 0, "\nPress to Quit...\n"); - getchar(); - } - else - { - shrLogEx(LOGBOTH | CLOSELOG, 0, "%s Exiting...\n", argv[0]); - } - fflush(stderr); - exit(iExitCode); -} - -#endif \ No newline at end of file diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 5ba29d57..d44c82c4 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -10,6 +10,8 @@ all: $(MAKE) -C fence $(MAKE) -C no_mf_ext $(MAKE) -C no_smem + $(MAKE) -C vecaddx + $(MAKE) -C sgemmx run-simx: $(MAKE) -C basic run-simx @@ -23,6 +25,8 @@ run-simx: $(MAKE) -C fence run-simx $(MAKE) -C no_mf_ext run-simx $(MAKE) -C no_smem run-simx + $(MAKE) -C vecaddx run-simx + $(MAKE) -C sgemmx run-simx run-rtlsim: $(MAKE) -C basic run-rtlsim @@ -36,6 +40,8 @@ run-rtlsim: $(MAKE) -C fence run-rtlsim $(MAKE) -C no_mf_ext run-rtlsim $(MAKE) -C no_smem run-rtlsim + $(MAKE) -C vecaddx run-rtlsim + $(MAKE) -C sgemmx run-rtlsim run-opae: $(MAKE) -C basic run-opae @@ -49,6 +55,8 @@ run-opae: $(MAKE) -C fence run-opae $(MAKE) -C no_mf_ext run-opae $(MAKE) -C no_smem run-opae + $(MAKE) -C vecaddx run-opae + $(MAKE) -C sgemmx run-opae clean: $(MAKE) -C basic clean @@ -62,6 +70,8 @@ clean: $(MAKE) -C fence clean $(MAKE) -C no_mf_ext clean $(MAKE) -C no_smem clean + $(MAKE) -C vecaddx clean + $(MAKE) -C sgemmx clean clean-all: $(MAKE) -C basic clean-all @@ -75,3 +85,5 @@ clean-all: $(MAKE) -C fence clean-all $(MAKE) -C no_mf_ext clean-all $(MAKE) -C no_smem clean-all + $(MAKE) -C vecaddx clean-all + $(MAKE) -C sgemmx clean-all diff --git a/tests/regression/basic/main.cpp b/tests/regression/basic/main.cpp index e79387b5..0f6f3bde 100755 --- a/tests/regression/basic/main.cpp +++ b/tests/regression/basic/main.cpp @@ -262,11 +262,8 @@ int main(int argc, char *argv[]) { // upload kernel argument std::cout << "upload kernel argument" << std::endl; - { - auto buf_ptr = (void*)staging_buf.data(); - memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); - RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); - } + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); std::cout << "run kernel test" << std::endl; RT_CHECK(run_kernel_test(kernel_arg, buf_size, num_points)); diff --git a/tests/regression/common.mk b/tests/regression/common.mk index 006d6668..6a858edc 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -1,16 +1,18 @@ XLEN ?= 32 +TOOLDIR ?= /opt + TARGET ?= opaesim XRT_SYN_DIR ?= ../../../hw/syn/xilinx/xrt XRT_DEVICE_INDEX ?= 0 ifeq ($(XLEN),64) -RISCV_TOOLCHAIN_PATH ?= /opt/riscv64-gnu-toolchain +RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv64-gnu-toolchain VX_CFLAGS += -march=rv64imafd -mabi=lp64d STARTUP_ADDR ?= 0x180000000 else -RISCV_TOOLCHAIN_PATH ?= /opt/riscv-gnu-toolchain +RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv-gnu-toolchain VX_CFLAGS += -march=rv32imaf -mabi=ilp32f STARTUP_ADDR ?= 0x80000000 endif @@ -23,7 +25,7 @@ VORTEX_KN_PATH ?= $(realpath ../../../kernel) FPGA_BIN_DIR ?= $(VORTEX_RT_PATH)/opae -LLVM_VORTEX ?= /opt/llvm-vortex +LLVM_VORTEX ?= $(TOOLDIR)/llvm-vortex LLVM_CFLAGS += --sysroot=$(RISCV_SYSROOT) LLVM_CFLAGS += --gcc-toolchain=$(RISCV_TOOLCHAIN_PATH) diff --git a/tests/regression/demo/common.h b/tests/regression/demo/common.h index e18b65a0..941983ac 100644 --- a/tests/regression/demo/common.h +++ b/tests/regression/demo/common.h @@ -3,6 +3,10 @@ #define KERNEL_ARG_DEV_MEM_ADDR 0x7ffff000 +#ifndef TYPE +#define TYPE float +#endif + typedef struct { uint32_t num_tasks; uint32_t task_size; @@ -11,4 +15,4 @@ typedef struct { uint64_t dst_addr; } kernel_arg_t; -#endif \ No newline at end of file +#endif diff --git a/tests/regression/demo/kernel.cpp b/tests/regression/demo/kernel.cpp index deb56169..49945440 100644 --- a/tests/regression/demo/kernel.cpp +++ b/tests/regression/demo/kernel.cpp @@ -4,11 +4,11 @@ #include "common.h" void kernel_body(int task_id, kernel_arg_t* __UNIFORM__ arg) { - uint32_t count = arg->task_size; - int32_t* src0_ptr = (int32_t*)arg->src0_addr; - int32_t* src1_ptr = (int32_t*)arg->src1_addr; - int32_t* dst_ptr = (int32_t*)arg->dst_addr; + auto src0_ptr = reinterpret_cast(arg->src0_addr); + auto src1_ptr = reinterpret_cast(arg->src1_addr); + auto dst_ptr = reinterpret_cast(arg->dst_addr); + uint32_t count = arg->task_size; uint32_t offset = task_id * count; for (uint32_t i = 0; i < count; ++i) { diff --git a/tests/regression/demo/main.cpp b/tests/regression/demo/main.cpp index dfe33377..f14f66c3 100644 --- a/tests/regression/demo/main.cpp +++ b/tests/regression/demo/main.cpp @@ -5,6 +5,8 @@ #include #include "common.h" +#define FLOAT_ULP 6 + #define RT_CHECK(_expr) \ do { \ int _ret = _expr; \ @@ -17,10 +19,61 @@ /////////////////////////////////////////////////////////////////////////////// +template +class Comparator {}; + +template <> +class Comparator { +public: + static const char* type_str() { + return "integer"; + } + static int generate() { + return rand(); + } + static bool compare(int a, int b, int index, int errors) { + if (a != b) { + if (errors < 100) { + printf("*** error: [%d] expected=%d, actual=%d\n", index, a, b); + } + return false; + } + return true; + } +}; + +template <> +class Comparator { +private: + union Float_t { float f; int i; }; +public: + static const char* type_str() { + return "float"; + } + static int generate() { + return static_cast(rand()) / RAND_MAX; + } + static bool compare(float a, float b, int index, int errors) { + union fi_t { float f; int32_t i; }; + fi_t fa, fb; + fa.f = a; + fb.f = b; + auto d = std::abs(fa.i - fb.i); + if (d > FLOAT_ULP) { + if (errors < 100) { + printf("*** error: [%d] expected=%f, actual=%f\n", index, a, b); + } + return false; + } + return true; + } +}; + const char* kernel_file = "kernel.bin"; -uint32_t count = 0; +uint32_t count = 16; vx_device_h device = nullptr; +std::vector source_data; std::vector staging_buf; kernel_arg_t kernel_arg = {}; @@ -79,13 +132,11 @@ int run_test(const kernel_arg_t& kernel_arg, std::cout << "verify result" << std::endl; { int errors = 0; - auto buf_ptr = (int32_t*)staging_buf.data(); + auto buf_ptr = (TYPE*)staging_buf.data(); for (uint32_t i = 0; i < num_points; ++i) { - int ref = i + i; - int cur = buf_ptr[i]; - if (cur != ref) { - std::cout << "error at result #" << std::dec << i - << std::hex << ": actual 0x" << cur << ", expected 0x" << ref << std::endl; + auto ref = source_data[2 * i + 0] + source_data[2 * i + 1]; + auto cur = buf_ptr[i]; + if (!Comparator::compare(cur, ref, i, errors)) { ++errors; } } @@ -103,9 +154,7 @@ int main(int argc, char *argv[]) { // parse command arguments parse_args(argc, argv); - if (count == 0) { - count = 1; - } + std::srand(50); // open device connection std::cout << "open device connection" << std::endl; @@ -118,8 +167,9 @@ int main(int argc, char *argv[]) { uint32_t num_tasks = num_cores * num_warps * num_threads; uint32_t num_points = count * num_tasks; - uint32_t buf_size = num_points * sizeof(int32_t); + uint32_t buf_size = num_points * sizeof(TYPE); + std::cout << "data type: " << Comparator::type_str() << std::endl; std::cout << "number of points: " << num_points << std::endl; std::cout << "buffer size: " << buf_size << " bytes" << std::endl; @@ -147,18 +197,21 @@ int main(int argc, char *argv[]) { // upload kernel argument std::cout << "upload kernel argument" << std::endl; - { - auto buf_ptr = (int*)staging_buf.data(); - memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); - RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + + // generate source data + source_data.resize(2 * num_points); + for (uint32_t i = 0; i < source_data.size(); ++i) { + source_data[i] = Comparator::generate(); } // upload source buffer0 { std::cout << "upload source buffer0" << std::endl; - auto buf_ptr = (int32_t*)staging_buf.data(); + auto buf_ptr = (TYPE*)staging_buf.data(); for (uint32_t i = 0; i < num_points; ++i) { - buf_ptr[i] = i-1; + buf_ptr[i] = source_data[2 * i + 0]; } RT_CHECK(vx_copy_to_dev(device, kernel_arg.src0_addr, staging_buf.data(), buf_size)); } @@ -166,23 +219,18 @@ int main(int argc, char *argv[]) { // upload source buffer1 { std::cout << "upload source buffer1" << std::endl; - auto buf_ptr = (int32_t*)staging_buf.data(); + auto buf_ptr = (TYPE*)staging_buf.data(); for (uint32_t i = 0; i < num_points; ++i) { - buf_ptr[i] = i+1; + buf_ptr[i] = source_data[2 * i + 1]; } RT_CHECK(vx_copy_to_dev(device, kernel_arg.src1_addr, staging_buf.data(), buf_size)); } // clear destination buffer - { - std::cout << "clear destination buffer" << std::endl; - auto buf_ptr = (int32_t*)staging_buf.data(); - for (uint32_t i = 0; i < num_points; ++i) { - buf_ptr[i] = 0xdeadbeef; - } - RT_CHECK(vx_copy_to_dev(device, kernel_arg.dst_addr, staging_buf.data(), buf_size)); - } - + std::cout << "clear destination buffer" << std::endl; + memset(staging_buf.data(), 0, num_points * sizeof(TYPE)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.dst_addr, staging_buf.data(), buf_size)); + // run tests std::cout << "run tests" << std::endl; RT_CHECK(run_test(kernel_arg, buf_size, num_points)); diff --git a/tests/regression/diverge/main.cpp b/tests/regression/diverge/main.cpp index 742f2419..d5de1bc1 100644 --- a/tests/regression/diverge/main.cpp +++ b/tests/regression/diverge/main.cpp @@ -233,11 +233,8 @@ int main(int argc, char *argv[]) { // upload kernel argument std::cout << "upload kernel argument" << std::endl; - { - auto buf_ptr = (int*)staging_buf.data(); - memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); - RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); - } + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); // upload source buffer { diff --git a/tests/regression/fence/main.cpp b/tests/regression/fence/main.cpp index d9f2920f..c9225edc 100644 --- a/tests/regression/fence/main.cpp +++ b/tests/regression/fence/main.cpp @@ -147,11 +147,8 @@ int main(int argc, char *argv[]) { // upload kernel argument std::cout << "upload kernel argument" << std::endl; - { - auto buf_ptr = (int*)staging_buf.data(); - memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); - RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); - } + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); // upload source buffer0 { diff --git a/tests/regression/io_addr/main.cpp b/tests/regression/io_addr/main.cpp index d4c74aad..0272bfbc 100644 --- a/tests/regression/io_addr/main.cpp +++ b/tests/regression/io_addr/main.cpp @@ -190,13 +190,10 @@ int main(int argc, char *argv[]) { staging_buf.resize(staging_buf_size); // upload kernel argument - { - std::cout << "upload kernel argument" << std::endl; - auto buf_ptr = (int*)staging_buf.data(); - memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); - RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); - } - + std::cout << "upload kernel argument" << std::endl; + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + // upload test address data { std::cout << "upload test address data" << std::endl; diff --git a/tests/regression/mstress/main.cpp b/tests/regression/mstress/main.cpp index ecc867bc..9b527126 100644 --- a/tests/regression/mstress/main.cpp +++ b/tests/regression/mstress/main.cpp @@ -236,13 +236,10 @@ int main(int argc, char *argv[]) { staging_buf.resize(staging_buf_size); // upload kernel argument - { - std::cout << "upload kernel argument" << std::endl; - auto buf_ptr = (int*)staging_buf.data(); - memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); - RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); - } - + std::cout << "upload kernel argument" << std::endl; + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + // upload source buffer0 { std::cout << "upload address buffer" << std::endl; diff --git a/tests/regression/no_mf_ext/main.cpp b/tests/regression/no_mf_ext/main.cpp index 7632dad1..e711b99a 100644 --- a/tests/regression/no_mf_ext/main.cpp +++ b/tests/regression/no_mf_ext/main.cpp @@ -136,13 +136,10 @@ int main(int argc, char *argv[]) { staging_buf.resize(alloc_size); // upload kernel argument - { - std::cout << "upload kernel argument" << std::endl; - auto buf_ptr = (int*)staging_buf.data(); - memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); - RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); - } - + std::cout << "upload kernel argument" << std::endl; + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + // upload source buffer0 { std::cout << "upload source buffer" << std::endl; diff --git a/tests/regression/no_smem/main.cpp b/tests/regression/no_smem/main.cpp index 8bb00389..53db0465 100644 --- a/tests/regression/no_smem/main.cpp +++ b/tests/regression/no_smem/main.cpp @@ -135,13 +135,10 @@ int main(int argc, char *argv[]) { uint32_t alloc_size = std::max(buf_size, sizeof(kernel_arg_t)); staging_buf.resize(alloc_size); - // upload kernel argument + // upload kernel argument std::cout << "upload kernel argument" << std::endl; - { - auto buf_ptr = (int*)staging_buf.data(); - memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); - RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); - } + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); // upload source buffer0 { diff --git a/tests/regression/printf/main.cpp b/tests/regression/printf/main.cpp index 4b13faad..3a920294 100644 --- a/tests/regression/printf/main.cpp +++ b/tests/regression/printf/main.cpp @@ -110,13 +110,10 @@ int main(int argc, char *argv[]) { staging_buf.resize(alloc_size); // upload kernel argument - { - std::cout << "upload kernel argument" << std::endl; - auto buf_ptr = (void*)staging_buf.data(); - memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); - RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); - } - + std::cout << "upload kernel argument" << std::endl; + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + // upload source buffer0 { std::cout << "upload source buffer" << std::endl; diff --git a/tests/regression/sgemmx/Makefile b/tests/regression/sgemmx/Makefile new file mode 100644 index 00000000..2e72b32e --- /dev/null +++ b/tests/regression/sgemmx/Makefile @@ -0,0 +1,9 @@ +PROJECT = sgemmx + +SRCS = main.cpp + +VX_SRCS = kernel.cpp + +OPTS ?= -n32 + +include ../common.mk \ No newline at end of file diff --git a/tests/regression/sgemmx/common.h b/tests/regression/sgemmx/common.h new file mode 100644 index 00000000..75cfc340 --- /dev/null +++ b/tests/regression/sgemmx/common.h @@ -0,0 +1,18 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#define KERNEL_ARG_DEV_MEM_ADDR 0x7ffff000 + +#ifndef TYPE +#define TYPE float +#endif + +typedef struct { + uint32_t num_tasks; + uint32_t size; + uint64_t A_addr; + uint64_t B_addr; + uint64_t C_addr; +} kernel_arg_t; + +#endif diff --git a/tests/regression/sgemmx/kernel.cpp b/tests/regression/sgemmx/kernel.cpp new file mode 100644 index 00000000..b0e8f69e --- /dev/null +++ b/tests/regression/sgemmx/kernel.cpp @@ -0,0 +1,41 @@ +#include +#include +#include +#include "common.h" + +inline char is_log2(uint32_t x) { + return ((x & (x-1)) == 0); +} + +inline uint32_t log2_fast(uint32_t x) { + return 31 - __builtin_clz (x); +} + +void kernel_body(uint32_t task_id, kernel_arg_t* __UNIFORM__ arg) { + auto A = reinterpret_cast(arg->A_addr); + auto B = reinterpret_cast(arg->B_addr); + auto C = reinterpret_cast(arg->C_addr); + auto size = arg->size; + + uint32_t row, col; + if (is_log2(size)) { + uint32_t log_size = log2_fast(size); + row = task_id >> log_size; + col = task_id & (size-1); + } else { + row = task_id / size; + col = task_id % size; + } + + TYPE sum (0); + for (int e = 0; e < size; ++e) { + sum += A[row * size + e] * B[e * size + col]; + } + C[row * size + col] = sum; +} + +int main() { + kernel_arg_t* arg = (kernel_arg_t*)KERNEL_ARG_DEV_MEM_ADDR; + vx_spawn_tasks(arg->num_tasks, (vx_spawn_tasks_cb)kernel_body, arg); + return 0; +} diff --git a/tests/regression/sgemmx/main.cpp b/tests/regression/sgemmx/main.cpp new file mode 100644 index 00000000..23008011 --- /dev/null +++ b/tests/regression/sgemmx/main.cpp @@ -0,0 +1,251 @@ +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define FLOAT_ULP 6 + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +template +class Comparator {}; + +template <> +class Comparator { +public: + static const char* type_str() { + return "integer"; + } + static int generate() { + return rand(); + } + static bool compare(int a, int b, int index, int errors) { + if (a != b) { + if (errors < 100) { + printf("*** error: [%d] expected=%d, actual=%d\n", index, a, b); + } + return false; + } + return true; + } +}; + +template <> +class Comparator { +public: + static const char* type_str() { + return "float"; + } + static int generate() { + return static_cast(rand()) / RAND_MAX; + } + static bool compare(float a, float b, int index, int errors) { + union fi_t { float f; int32_t i; }; + fi_t fa, fb; + fa.f = a; + fb.f = b; + auto d = std::abs(fa.i - fb.i); + if (d > FLOAT_ULP) { + if (errors < 100) { + printf("*** error: [%d] expected=%f, actual=%f\n", index, a, b); + } + return false; + } + return true; + } +}; + +static void matmul_cpu(TYPE* out, const TYPE* A, const TYPE* B, uint32_t width, uint32_t height) { + for (uint32_t row = 0; row < height; ++row) { + for (uint32_t col = 0; col < width; ++col) { + TYPE sum(0); + for (uint32_t e = 0; e < width; ++e) { + sum += A[row * width + e] * B[e * width + col]; + } + out[row * width + col] = sum; + } + } +} + +const char* kernel_file = "kernel.bin"; +uint32_t size = 32; + +vx_device_h device = nullptr; +std::vector staging_buf; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n size] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h?")) != -1) { + switch (c) { + case 'n': + size = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + case '?': { + show_usage(); + exit(0); + } break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + vx_mem_free(device, kernel_arg.A_addr); + vx_mem_free(device, kernel_arg.B_addr); + vx_mem_free(device, kernel_arg.C_addr); + vx_dev_close(device); + } +} + +int main(int argc, char *argv[]) { + // parse command arguments + parse_args(argc, argv); + + std::srand(50); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_dev_open(&device)); + + uint32_t num_points = size * size; + uint32_t buf_size = num_points * sizeof(TYPE); + + std::cout << "data type: " << Comparator::type_str() << std::endl; + std::cout << "matrix size: " << size << "x" << size << std::endl; + std::cout << "buffer size: " << buf_size << " bytes" << std::endl; + + // upload program + std::cout << "upload program" << std::endl; + RT_CHECK(vx_upload_kernel_file(device, kernel_file)); + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.A_addr)); + RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.B_addr)); + RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.C_addr)); + + kernel_arg.num_tasks = num_points; + kernel_arg.size = size; + + std::cout << "dev_src0=0x" << std::hex << kernel_arg.A_addr << std::endl; + std::cout << "dev_src1=0x" << std::hex << kernel_arg.B_addr << std::endl; + std::cout << "dev_dst=0x" << std::hex << kernel_arg.C_addr << std::endl; + + // allocate staging buffer + std::cout << "allocate staging buffer" << std::endl; + uint32_t alloc_size = std::max(buf_size, sizeof(kernel_arg_t)); + staging_buf.resize(alloc_size); + + // upload kernel argument + std::cout << "upload kernel argument" << std::endl; + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + + // generate source data + std::vector src_A(num_points); + std::vector src_B(num_points); + std::vector refs(num_points); + for (uint32_t i = 0; i < num_points; ++i) { + auto a = static_cast(std::rand()) / RAND_MAX; + auto b = static_cast(std::rand()) / RAND_MAX; + src_A[i] = static_cast(a * size); + src_B[i] = static_cast(b * size); + } + matmul_cpu(refs.data(), src_A.data(), src_B.data(), size, size); + + // upload source buffer0 + { + std::cout << "upload source buffer0" << std::endl; + auto buf_ptr = (TYPE*)staging_buf.data(); + for (uint32_t i = 0; i < num_points; ++i) { + buf_ptr[i] = src_A[i]; + } + RT_CHECK(vx_copy_to_dev(device, kernel_arg.A_addr, staging_buf.data(), buf_size)); + } + + // upload source buffer1 + { + std::cout << "upload source buffer1" << std::endl; + auto buf_ptr = (TYPE*)staging_buf.data(); + for (uint32_t i = 0; i < num_points; ++i) { + buf_ptr[i] = src_B[i]; + } + RT_CHECK(vx_copy_to_dev(device, kernel_arg.B_addr, staging_buf.data(), buf_size)); + } + + // clear destination buffer + std::cout << "clear destination buffer" << std::endl; + memset(staging_buf.data(), 0, num_points * sizeof(TYPE)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.C_addr, staging_buf.data(), buf_size)); + + auto time_start = std::chrono::high_resolution_clock::now(); + + // start device + std::cout << "start device" << std::endl; + RT_CHECK(vx_start(device)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT)); + + auto time_end = std::chrono::high_resolution_clock::now(); + double elapsed = std::chrono::duration_cast(time_end - time_start).count(); + printf("Elapsed time: %lg ms\n", elapsed); + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.C_addr, buf_size)); + + // verify result + std::cout << "verify result" << std::endl; + { + int errors = 0; + auto buf_ptr = (TYPE*)staging_buf.data(); + for (uint32_t i = 0; i < refs.size(); ++i) { + auto ref = refs[i]; + auto cur = buf_ptr[i]; + if (!Comparator::compare(cur, ref, i, errors)) { + ++errors; + } + } + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + } + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + std::cout << "PASSED!" << std::endl; + + return 0; +} \ No newline at end of file diff --git a/tests/regression/sort/common.h b/tests/regression/sort/common.h index 492e03c6..92ceeb91 100644 --- a/tests/regression/sort/common.h +++ b/tests/regression/sort/common.h @@ -3,11 +3,7 @@ #define KERNEL_ARG_DEV_MEM_ADDR 0x7ffff000 -#define FP_ENABLE - -#ifdef FP_ENABLE -#define TYPE float -#else +#ifndef TYPE #define TYPE int #endif @@ -17,4 +13,4 @@ typedef struct { uint64_t dst_addr; } kernel_arg_t; -#endif \ No newline at end of file +#endif diff --git a/tests/regression/sort/kernel.cpp b/tests/regression/sort/kernel.cpp index 0cd7074e..2e9d3453 100644 --- a/tests/regression/sort/kernel.cpp +++ b/tests/regression/sort/kernel.cpp @@ -5,14 +5,14 @@ void kernel_body(int task_id, kernel_arg_t* __UNIFORM__ arg) { uint32_t num_points = arg->num_points; - TYPE* src_ptr = (TYPE*)arg->src_addr; - TYPE* dst_ptr = (TYPE*)arg->dst_addr; + auto src_ptr = (TYPE*)arg->src_addr; + auto dst_ptr = (TYPE*)arg->dst_addr; - TYPE ref_value = src_ptr[task_id]; + auto ref_value = src_ptr[task_id]; uint32_t pos = 0; for (uint32_t i = 0; i < num_points; ++i) { - TYPE cur_value = src_ptr[i]; + auto cur_value = src_ptr[i]; pos += (cur_value < ref_value) || ((cur_value == ref_value) && (i < task_id)); } dst_ptr[pos] = ref_value; diff --git a/tests/regression/sort/main.cpp b/tests/regression/sort/main.cpp index 59796f73..38d5d4d4 100644 --- a/tests/regression/sort/main.cpp +++ b/tests/regression/sort/main.cpp @@ -66,8 +66,8 @@ void gen_input_data(uint32_t num_points) { src_data.resize(num_points); for (uint32_t i = 0; i < num_points; ++i) { - float r = static_cast(std::rand()) / RAND_MAX; - TYPE value = r * num_points; + auto r = static_cast(std::rand()) / RAND_MAX; + auto value = static_cast(r * num_points); src_data[i] = value; std::cout << std::dec << i << ": value=" << value << std::endl; } @@ -172,19 +172,16 @@ int main(int argc, char *argv[]) { { std::cout << "allocate staging buffer" << std::endl; uint32_t staging_buf_size = std::max(src_buf_size, - std::max(dst_buf_size, - sizeof(kernel_arg_t))); + std::max(dst_buf_size, + sizeof(kernel_arg_t))); staging_buf.resize(staging_buf_size); } // upload kernel argument - { - std::cout << "upload kernel argument" << std::endl; - auto buf_ptr = staging_buf.data(); - memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); - RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); - } - + std::cout << "upload kernel argument" << std::endl; + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + // upload source buffer { std::cout << "upload source buffer" << std::endl; diff --git a/tests/regression/vecaddx/Makefile b/tests/regression/vecaddx/Makefile new file mode 100644 index 00000000..af43d3c7 --- /dev/null +++ b/tests/regression/vecaddx/Makefile @@ -0,0 +1,9 @@ +PROJECT = vecaddx + +SRCS = main.cpp + +VX_SRCS = kernel.cpp + +OPTS ?= -n64 + +include ../common.mk \ No newline at end of file diff --git a/tests/regression/vecaddx/common.h b/tests/regression/vecaddx/common.h new file mode 100644 index 00000000..2b8f164a --- /dev/null +++ b/tests/regression/vecaddx/common.h @@ -0,0 +1,17 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#define KERNEL_ARG_DEV_MEM_ADDR 0x7ffff000 + +#ifndef TYPE +#define TYPE float +#endif + +typedef struct { + uint32_t num_points; + uint64_t src0_addr; + uint64_t src1_addr; + uint64_t dst_addr; +} kernel_arg_t; + +#endif diff --git a/tests/regression/vecaddx/kernel.cpp b/tests/regression/vecaddx/kernel.cpp new file mode 100644 index 00000000..6ed42164 --- /dev/null +++ b/tests/regression/vecaddx/kernel.cpp @@ -0,0 +1,18 @@ +#include +#include +#include +#include "common.h" + +void kernel_body(int task_id, kernel_arg_t* __UNIFORM__ arg) { + auto src0_ptr = reinterpret_cast(arg->src0_addr); + auto src1_ptr = reinterpret_cast(arg->src1_addr); + auto dst_ptr = reinterpret_cast(arg->dst_addr); + + dst_ptr[task_id] = src0_ptr[task_id] + src1_ptr[task_id]; +} + +int main() { + kernel_arg_t* arg = (kernel_arg_t*)KERNEL_ARG_DEV_MEM_ADDR; + vx_spawn_tasks(arg->num_points, (vx_spawn_tasks_cb)kernel_body, arg); + return 0; +} diff --git a/tests/regression/vecaddx/main.cpp b/tests/regression/vecaddx/main.cpp new file mode 100644 index 00000000..117f3470 --- /dev/null +++ b/tests/regression/vecaddx/main.cpp @@ -0,0 +1,246 @@ +#include +#include +#include +#include +#include +#include "common.h" + +#define FLOAT_ULP 6 + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +template +class Comparator {}; + +template <> +class Comparator { +public: + static const char* type_str() { + return "integer"; + } + static int generate() { + return rand(); + } + static bool compare(int a, int b, int index, int errors) { + if (a != b) { + if (errors < 100) { + printf("*** error: [%d] expected=%d, actual=%d\n", index, a, b); + } + return false; + } + return true; + } +}; + +template <> +class Comparator { +private: + union Float_t { float f; int i; }; +public: + static const char* type_str() { + return "float"; + } + static int generate() { + return static_cast(rand()) / RAND_MAX; + } + static bool compare(float a, float b, int index, int errors) { + union fi_t { float f; int32_t i; }; + fi_t fa, fb; + fa.f = a; + fb.f = b; + auto d = std::abs(fa.i - fb.i); + if (d > FLOAT_ULP) { + if (errors < 100) { + printf("*** error: [%d] expected=%f, actual=%f\n", index, a, b); + } + return false; + } + return true; + } +}; + +const char* kernel_file = "kernel.bin"; +uint32_t size = 16; + +vx_device_h device = nullptr; +std::vector source_data; +std::vector staging_buf; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h?")) != -1) { + switch (c) { + case 'n': + size = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + case '?': { + show_usage(); + exit(0); + } break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + vx_mem_free(device, kernel_arg.src0_addr); + vx_mem_free(device, kernel_arg.src1_addr); + vx_mem_free(device, kernel_arg.dst_addr); + vx_dev_close(device); + } +} + +int run_test(const kernel_arg_t& kernel_arg, + uint32_t buf_size, + uint32_t num_points) { + // start device + std::cout << "start device" << std::endl; + RT_CHECK(vx_start(device)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT)); + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.dst_addr, buf_size)); + + // verify result + std::cout << "verify result" << std::endl; + { + int errors = 0; + auto buf_ptr = (TYPE*)staging_buf.data(); + for (uint32_t i = 0; i < num_points; ++i) { + auto ref = source_data[2 * i + 0] + source_data[2 * i + 1]; + auto cur = buf_ptr[i]; + if (!Comparator::compare(cur, ref, i, errors)) { + ++errors; + } + } + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + } + + return 0; +} + +int main(int argc, char *argv[]) { + // parse command arguments + parse_args(argc, argv); + + std::srand(50); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_dev_open(&device)); + + uint64_t num_cores, num_warps, num_threads; + RT_CHECK(vx_dev_caps(device, VX_CAPS_NUM_CORES, &num_cores)); + RT_CHECK(vx_dev_caps(device, VX_CAPS_NUM_WARPS, &num_warps)); + RT_CHECK(vx_dev_caps(device, VX_CAPS_NUM_THREADS, &num_threads)); + std::cout << "number of cores: " << num_cores << std::endl; + std::cout << "number of warps: " << num_warps << std::endl; + std::cout << "number of threads: " << num_threads << std::endl; + + uint32_t num_points = size; + uint32_t buf_size = num_points * sizeof(TYPE); + + std::cout << "number of points: " << num_points << std::endl; + std::cout << "data type: " << Comparator::type_str() << std::endl; + std::cout << "buffer size: " << buf_size << " bytes" << std::endl; + + // upload program + std::cout << "upload program" << std::endl; + RT_CHECK(vx_upload_kernel_file(device, kernel_file)); + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.src0_addr)); + RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.src1_addr)); + RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.dst_addr)); + + kernel_arg.num_points = num_points; + + std::cout << "dev_src0=0x" << std::hex << kernel_arg.src0_addr << std::endl; + std::cout << "dev_src1=0x" << std::hex << kernel_arg.src1_addr << std::endl; + std::cout << "dev_dst=0x" << std::hex << kernel_arg.dst_addr << std::endl; + + // allocate staging buffer + std::cout << "allocate staging buffer" << std::endl; + uint32_t alloc_size = std::max(buf_size, sizeof(kernel_arg_t)); + staging_buf.resize(alloc_size); + + // upload kernel argument + std::cout << "upload kernel argument" << std::endl; + memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + + // generate source data + source_data.resize(2 * num_points); + for (uint32_t i = 0; i < source_data.size(); ++i) { + source_data[i] = Comparator::generate(); + } + + // upload source buffer0 + { + std::cout << "upload source buffer0" << std::endl; + auto buf_ptr = (TYPE*)staging_buf.data(); + for (uint32_t i = 0; i < num_points; ++i) { + buf_ptr[i] = source_data[2 * i + 0]; + } + RT_CHECK(vx_copy_to_dev(device, kernel_arg.src0_addr, staging_buf.data(), buf_size)); + } + + // upload source buffer1 + { + std::cout << "upload source buffer1" << std::endl; + auto buf_ptr = (TYPE*)staging_buf.data(); + for (uint32_t i = 0; i < num_points; ++i) { + buf_ptr[i] = source_data[2 * i + 1]; + } + RT_CHECK(vx_copy_to_dev(device, kernel_arg.src1_addr, staging_buf.data(), buf_size)); + } + + // clear destination buffer + std::cout << "clear destination buffer" << std::endl; + memset(staging_buf.data(), 0, num_points * sizeof(TYPE)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.dst_addr, staging_buf.data(), buf_size)); + + // run tests + std::cout << "run tests" << std::endl; + RT_CHECK(run_test(kernel_arg, buf_size, num_points)); + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + std::cout << "PASSED!" << std::endl; + + return 0; +} \ No newline at end of file diff --git a/tests/unittest/common.mk b/tests/unittest/common.mk new file mode 100644 index 00000000..11add105 --- /dev/null +++ b/tests/unittest/common.mk @@ -0,0 +1,30 @@ +VORTEX_RT_PATH ?= $(realpath ../../../runtime) + +CXXFLAGS += -std=c++11 -Wall -Wextra -pedantic -Wfatal-errors + +CXXFLAGS += -I$(VORTEX_RT_PATH)/common + +# Debugigng +ifdef DEBUG + CXXFLAGS += -g -O0 +else + CXXFLAGS += -O2 -DNDEBUG +endif + +all: $(PROJECT) + +$(PROJECT): $(SRCS) + $(CXX) $(CXXFLAGS) $^ $(LDFLAGS) -o $@ + +run: + ./$(PROJECT) + +clean: + rm -rf $(PROJECT) *.o .depend + +clean-all: clean + rm -rf *.elf *.bin *.dump + +ifneq ($(MAKECMDGOALS),clean) + -include .depend +endif diff --git a/tests/unittest/vx_malloc/Makefile b/tests/unittest/vx_malloc/Makefile index 2d604620..2036fcde 100644 --- a/tests/unittest/vx_malloc/Makefile +++ b/tests/unittest/vx_malloc/Makefile @@ -1,34 +1,5 @@ -VORTEX_RT_PATH ?= $(realpath ../../../runtime) - -CXXFLAGS += -std=c++11 -Wall -Wextra -pedantic -Wfatal-errors - -CXXFLAGS += -I$(VORTEX_RT_PATH)/common - -# Debugigng -ifdef DEBUG - CXXFLAGS += -g -O0 -else - CXXFLAGS += -O2 -DNDEBUG -endif - PROJECT = vx_malloc SRCS = main.cpp -all: $(PROJECT) - -$(PROJECT): $(SRCS) - $(CXX) $(CXXFLAGS) $^ $(LDFLAGS) -o $@ - -run: - ./$(PROJECT) - -clean: - rm -rf $(PROJECT) *.o .depend - -clean-all: clean - rm -rf *.elf *.bin *.dump - -ifneq ($(MAKECMDGOALS),clean) - -include .depend -endif \ No newline at end of file +include ../common.mk