add cnn_trt lib.

src/detection/detection_lidar_cnn_segmentation_trt/lib/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 lewes6369
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

src/detection/detection_lidar_cnn_segmentation_trt/lib/include/TrtNet.hpp

@@ -0,0 +1,115 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2018 lewes6369
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef TRTNET_HPP_
+#define TRTNET_HPP_
+
+#include "Utils.hpp"
+
+#include <NvCaffeParser.h>
+#include <NvInferPlugin.h>
+
+#include <algorithm>
+#include <fstream>
+#include <memory>
+#include <numeric>
+#include <string>
+#include <vector>
+
+namespace Tn
+{
+enum class RUN_MODE { FLOAT32 = 0, FLOAT16 = 1, INT8 = 2 };
+
+class trtNet
+{
+public:
+  // Load from engine file
+  explicit trtNet(const std::string & engineFile);
+
+  ~trtNet()
+  {
+    // Release the stream and the buffers
+    cudaStreamSynchronize(mTrtCudaStream);
+    cudaStreamDestroy(mTrtCudaStream);
+    for (auto & item : mTrtCudaBuffer) {
+      cudaFree(item);
+    }
+
+    if (!mTrtRunTime) {
+      delete mTrtRunTime;
+    }
+    if (!mTrtContext) {
+      delete mTrtContext;
+    }
+    if (!mTrtEngine) {
+      delete mTrtEngine;
+    }
+  }
+
+  void saveEngine(std::string fileName)
+  {
+    if (mTrtEngine) {
+      nvinfer1::IHostMemory * data = mTrtEngine->serialize();
+      std::ofstream file;
+      file.open(fileName, std::ios::binary | std::ios::out);
+      if (!file.is_open()) {
+        std::cout << "read create engine file" << fileName << " failed" << std::endl;
+        return;
+      }
+
+      file.write((const char *)data->data(), data->size());
+      file.close();
+    }
+  }
+
+  void doInference(const void * inputData, void * outputData);
+
+  inline size_t getInputSize()
+  {
+    return std::accumulate(
+      mTrtBindBufferSize.begin(), mTrtBindBufferSize.begin() + mTrtInputCount, 0);
+  }
+
+  inline size_t getOutputSize()
+  {
+    return std::accumulate(
+      mTrtBindBufferSize.begin() + mTrtInputCount, mTrtBindBufferSize.end(), 0);
+  }
+
+private:
+  void InitEngine();
+
+  nvinfer1::IExecutionContext * mTrtContext;
+  nvinfer1::ICudaEngine * mTrtEngine;
+  nvinfer1::IRuntime * mTrtRunTime;
+  cudaStream_t mTrtCudaStream;
+  Profiler mTrtProfiler;
+  RUN_MODE mTrtRunMode;
+
+  std::vector<void *> mTrtCudaBuffer;
+  std::vector<int64_t> mTrtBindBufferSize;
+  int mTrtInputCount;
+};
+}  // namespace Tn
+
+#endif  // TRTNET_HPP_

src/detection/detection_lidar_cnn_segmentation_trt/lib/include/Utils.hpp

@@ -0,0 +1,134 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2018 lewes6369
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef UTILS_HPP_
+#define UTILS_HPP_
+
+#include <NvInferPlugin.h>
+#include <cudnn.h>
+
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+#ifndef CUDA_CHECK
+
+#define CUDA_CHECK(callstr)                                                              \
+  {                                                                                      \
+    cudaError_t error_code = callstr;                                                    \
+    if (error_code != cudaSuccess) {                                                     \
+      std::cerr << "CUDA error " << error_code << " at " << __FILE__ << ":" << __LINE__; \
+      assert(0);                                                                         \
+    }                                                                                    \
+  }
+
+#endif
+
+namespace Tn
+{
+class Profiler : public nvinfer1::IProfiler
+{
+public:
+  void printLayerTimes(int iterationsTimes)
+  {
+    float totalTime = 0;
+    for (size_t i = 0; i < mProfile.size(); i++) {
+      printf("%-40.40s %4.3fms\n", mProfile[i].first.c_str(), mProfile[i].second / iterationsTimes);
+      totalTime += mProfile[i].second;
+    }
+    printf("Time over all layers: %4.3f\n", totalTime / iterationsTimes);
+  }
+
+private:
+  typedef std::pair<std::string, float> Record;
+  std::vector<Record> mProfile;
+
+  void reportLayerTime(const char * layerName, float ms) noexcept override
+  {
+    auto record = std::find_if(
+      mProfile.begin(), mProfile.end(), [&](const Record & r) { return r.first == layerName; });
+    if (record == mProfile.end()) {
+      mProfile.push_back(std::make_pair(layerName, ms));
+    } else {
+      record->second += ms;
+    }
+  }
+};
+
+// Logger for TensorRT info/warning/errors
+class Logger : public nvinfer1::ILogger
+{
+public:
+  Logger() : Logger(Severity::kWARNING) {}
+
+  explicit Logger(Severity severity) : reportableSeverity(severity) {}
+
+  void log(Severity severity, const char * msg) noexcept override
+  {
+    // suppress messages with severity enum value greater than the reportable
+    if (severity > reportableSeverity) {
+      return;
+    }
+
+    switch (severity) {
+      case Severity::kINTERNAL_ERROR:
+        std::cerr << "INTERNAL_ERROR: ";
+        break;
+      case Severity::kERROR:
+        std::cerr << "ERROR: ";
+        break;
+      case Severity::kWARNING:
+        std::cerr << "WARNING: ";
+        break;
+      case Severity::kINFO:
+        std::cerr << "INFO: ";
+        break;
+      default:
+        std::cerr << "UNKNOWN: ";
+        break;
+    }
+    std::cerr << msg << std::endl;
+  }
+
+  Severity reportableSeverity{Severity::kWARNING};
+};
+
+template <typename T>
+void write(char *& buffer, const T & val)
+{
+  *reinterpret_cast<T *>(buffer) = val;
+  buffer += sizeof(T);
+}
+
+template <typename T>
+void read(const char *& buffer, T & val)
+{
+  val = *reinterpret_cast<const T *>(buffer);
+  buffer += sizeof(T);
+}
+}  // namespace Tn
+
+#endif  // UTILS_HPP_

src/detection/detection_lidar_cnn_segmentation_trt/lib/src/TrtNet.cpp

@@ -0,0 +1,152 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2018 lewes6369
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <TrtNet.hpp>
+
+#include <cublas_v2.h>
+#include <cudnn.h>
+#include <string.h>
+#include <time.h>
+
+#include <cassert>
+#include <chrono>
+#include <iostream>
+#include <sstream>
+#include <unordered_map>
+
+using namespace nvinfer1;        // NOLINT
+using namespace nvcaffeparser1;  // NOLINT
+using namespace plugin;          // NOLINT
+
+static Tn::Logger gLogger;
+
+inline void * safeCudaMalloc(size_t memSize)
+{
+  void * deviceMem;
+  CUDA_CHECK(cudaMalloc(&deviceMem, memSize));
+  if (deviceMem == nullptr) {
+    std::cerr << "Out of memory" << std::endl;
+    exit(1);
+  }
+  return deviceMem;
+}
+
+inline int64_t volume(const nvinfer1::Dims & d)
+{
+  return std::accumulate(d.d, d.d + d.nbDims, 1, std::multiplies<int64_t>());
+}
+
+inline unsigned int getElementSize(nvinfer1::DataType t)
+{
+  switch (t) {
+    case nvinfer1::DataType::kINT32:
+      return 4;
+    case nvinfer1::DataType::kFLOAT:
+      return 4;
+    case nvinfer1::DataType::kHALF:
+      return 2;
+    case nvinfer1::DataType::kINT8:
+      return 1;
+    default:
+      throw std::runtime_error("Invalid DataType.");
+      return 0;
+  }
+}
+
+namespace Tn
+{
+trtNet::trtNet(const std::string & engineFile)
+: mTrtContext(nullptr),
+  mTrtEngine(nullptr),
+  mTrtRunTime(nullptr),
+  mTrtRunMode(RUN_MODE::FLOAT32),
+  mTrtInputCount(0)
+{
+  using namespace std;  // NOLINT
+  fstream file;
+
+  file.open(engineFile, ios::binary | ios::in);
+  if (!file.is_open()) {
+    cout << "read engine file" << engineFile << " failed" << endl;
+    return;
+  }
+  file.seekg(0, ios::end);
+  int length = file.tellg();
+  file.seekg(0, ios::beg);
+  std::unique_ptr<char[]> data(new char[length]);
+  file.read(data.get(), length);
+  file.close();
+
+  mTrtRunTime = createInferRuntime(gLogger);
+  assert(mTrtRunTime != nullptr);
+  mTrtEngine = mTrtRunTime->deserializeCudaEngine(data.get(), length);
+  assert(mTrtEngine != nullptr);
+
+  InitEngine();
+}
+
+void trtNet::InitEngine()
+{
+  const int maxBatchSize = 1;
+  mTrtContext = mTrtEngine->createExecutionContext();
+  assert(mTrtContext != nullptr);
+  mTrtContext->setProfiler(&mTrtProfiler);
+
+  int nbBindings = mTrtEngine->getNbBindings();
+
+  mTrtCudaBuffer.resize(nbBindings);
+  mTrtBindBufferSize.resize(nbBindings);
+  for (int i = 0; i < nbBindings; ++i) {
+    Dims dims = mTrtEngine->getBindingDimensions(i);
+    DataType dtype = mTrtEngine->getBindingDataType(i);
+    int64_t totalSize = volume(dims) * maxBatchSize * getElementSize(dtype);
+    mTrtBindBufferSize[i] = totalSize;
+    mTrtCudaBuffer[i] = safeCudaMalloc(totalSize);
+    if (mTrtEngine->bindingIsInput(i)) {
+      mTrtInputCount++;
+    }
+  }
+
+  CUDA_CHECK(cudaStreamCreate(&mTrtCudaStream));
+}
+
+void trtNet::doInference(const void * inputData, void * outputData)
+{
+  static const int batchSize = 1;
+  assert(mTrtInputCount == 1);
+
+  int inputIndex = 0;
+  CUDA_CHECK(cudaMemcpyAsync(
+    mTrtCudaBuffer[inputIndex], inputData, mTrtBindBufferSize[inputIndex], cudaMemcpyHostToDevice,
+    mTrtCudaStream));
+
+  mTrtContext->execute(batchSize, &mTrtCudaBuffer[inputIndex]);
+
+  for (size_t bindingIdx = mTrtInputCount; bindingIdx < mTrtBindBufferSize.size(); ++bindingIdx) {
+    auto size = mTrtBindBufferSize[bindingIdx];
+    CUDA_CHECK(cudaMemcpyAsync(
+      outputData, mTrtCudaBuffer[bindingIdx], size, cudaMemcpyDeviceToHost, mTrtCudaStream));
+  }
+}
+}  // namespace Tn