Support hardware encoding

Dockerfile

@@ -13,7 +13,7 @@
 # Host Usage: (Interactive for Debug)
 # sudo docker run --gpus all --rm -it --network="host" --entrypoint=/bin/bash image-rigel
 
-FROM nvidia/cudagl:10.1-base-ubuntu18.04
+FROM nvidia/cudagl:11.1-devel-ubuntu18.04
 MAINTAINER Keitaro Oguri "ogukei256@gmail.com"
 
 # GPU Driver
@@ -90,19 +90,29 @@ USER root
 RUN apt-get update && apt-get install -qy lsb-release sudo
 RUN /home/user/webrtc-checkout/src/build/install-build-deps.sh --no-prompt --no-chromeos-fonts
 
-# Build libwebrtc (M75)
+# Build libwebrtc (M87)
 # http://webrtc.github.io/webrtc-org/native-code/development/
 USER user
 WORKDIR /home/user/webrtc-checkout/src
-RUN gclient sync -r branch-heads/m75
-RUN gn gen out/Default --args='target_os="linux" is_debug=false rtc_include_tests=false is_component_build=false use_rtti=true rtc_use_dummy_audio_file_devices=true'
+RUN gclient sync -r 69202b2a57b8b7f7046dc26930aafd6f779a152e
+RUN gn gen out/Default --args='target_os="linux" is_debug=false rtc_include_tests=false is_component_build=false use_rtti=true rtc_use_dummy_audio_file_devices=true rtc_use_h264=true'
 RUN ninja -C out/Default
 
 # Install Golang
 USER root
 RUN apt-get install -qy golang-1.10
 ENV PATH $PATH:/usr/lib/go-1.10/bin
 
+# libnvidia-encode.so
+USER root
+ENV NVIDIA_DRIVER_CAPABILITIES compute,graphics,video,utility
+# stubs
+COPY ./third_party/video_codec_sdk/lib/linux_x86_64/libnvcuvid.so /usr/lib/x86_64-linux-gnu/libnvcuvid.so
+COPY ./third_party/video_codec_sdk/lib/linux_x86_64/libnvidia-encode.so /usr/lib/x86_64-linux-gnu/libnvidia-encode.so
+
+# Debug
+RUN apt-get install -qy gdb valgrind
+
 # Build Belt
 USER user
 WORKDIR /home/user
@@ -120,6 +130,8 @@ WORKDIR /home/user/rigel
 ENV WEBRTC_ROOT /home/user/webrtc-checkout/src
 RUN make -j
 
+User root
+
 # Run Setup
 WORKDIR /home/user/rigel
 EXPOSE 8080

Makefile

@@ -31,10 +31,13 @@ TARGET=main
 
 BUILD_DIR=build
 SOURCE_DIR=src
+THIRD_PARTY_DIR=third_party
 
+INCLUDE_DIR_CUDA=/usr/local/cuda/include
 INCLUDES=-I$(WEBRTC_ROOT) \
 	-I$(WEBRTC_ROOT)/third_party/abseil-cpp \
 	-I$(WEBRTC_ROOT)/third_party/libyuv/include \
+	-I$(INCLUDE_DIR_CUDA) \
 	-I$(SOURCE_DIR)
 
 ISYSTEM_LIBCPP=-isystem$(WEBRTC_ROOT)/buildtools/third_party/libc++/trunk/include
@@ -47,17 +50,29 @@ CXXFLAGS=-Wno-macro-redefined \
 	-DWEBRTC_POSIX \
 	-D_LIBCPP_ABI_UNSTABLE \
 	-fno-lto \
-	-std=c++11 \
+	-std=c++14 \
 	-nostdinc++ \
 	$(ISYSTEM_LIBCPP) \
 	$(INCLUDES)
 
 LD=$(CXX)
 LDFLAGS_LINUX=-ldl -lX11
 LDFLAGS_VULKAN=-lvulkan
-LDFLAGS=-lpthread $(LDFLAGS_LINUX) $(LDFLAGS_VULKAN)
-
+LDFLAGS_CUDA=-lcuda
+LDFLAGS_CUDART=-lcudart 
+# stubs
+LDFLAGS_NVIDIA_ENCODE=-lnvidia-encode
+LDFLAGS_NVCUVID=-lnvcuvid
+# cuda
+LDFLAGS_CUDA_DIR=-L/usr/local/cuda/lib64
+# LDFLAGS
+LDFLAGS=-lpthread $(LDFLAGS_LINUX) $(LDFLAGS_VULKAN) \
+	$(LDFLAGS_CUDA) $(LDFLAGS_CUDART) $(LDFLAGS_NVIDIA_ENCODE) $(LDFLAGS_NVCUVID) \
+	$(LDFLAGS_CUDA_DIR)
+
+# WebRTC
 LIBWEBRTC_A=$(BUILD_DIR)/libwebrtc.a
+# LIBS
 LIBS=$(LIBWEBRTC_A)
 
 AR=$(WEBRTC_ROOT)/third_party/llvm-build/Release+Asserts/bin/llvm-ar

src/capture_frame.cc

@@ -0,0 +1,18 @@
+
+#include "capture_frame.h"
+#include <stdexcept>
+
+namespace rigel {
+
+NativeVideoFrameBuffer::NativeVideoFrameBuffer(int width, int height)
+    : width_(width), height_(height), device_pointer_(nullptr), pitch_(0)
+{
+
+}
+
+rtc::scoped_refptr<webrtc::I420BufferInterface> NativeVideoFrameBuffer::ToI420() {
+  throw std::runtime_error("NativeVideoFrameBuffer::ToI420() I420 format not supported");
+}
+
+}  // namespace rigel
+

src/capture_frame.h

@@ -0,0 +1,54 @@
+
+
+#ifndef RIGEL_RTC_CAPTURE_FRAME_H_
+#define RIGEL_RTC_CAPTURE_FRAME_H_
+
+#include <memory>
+
+// @see https://chromium.googlesource.com/external/webrtc/+/branch-heads/m75/api/video/video_frame_buffer.h
+#include "api/video/video_frame_buffer.h"
+#include "api/scoped_refptr.h"
+#include "rtc_base/ref_count.h"
+
+namespace rigel {
+
+class NativeVideoFrameBuffer : public webrtc::VideoFrameBuffer {
+ public:
+  explicit NativeVideoFrameBuffer(int width, int height);
+
+  void *DevicePointer() const { return device_pointer_;}
+  int Pitch() const { return pitch_; }
+
+  void Configure(void *dp, int pitch) {
+    device_pointer_ = dp;
+    pitch_ = pitch;
+  }
+
+  // This function specifies in what pixel format the data is stored in.
+  virtual webrtc::VideoFrameBuffer::Type type() const override {
+    return webrtc::VideoFrameBuffer::Type::kNative;
+  }
+  // The resolution of the frame in pixels. For formats where some planes are
+  // subsampled, this is the highest-resolution plane.
+  virtual int width() const override { return width_; }
+  virtual int height() const override { return height_; }
+
+  // Returns a memory-backed frame buffer in I420 format. If the pixel data is
+  // in another format, a conversion will take place. All implementations must
+  // provide a fallback to I420 for compatibility with e.g. the internal WebRTC
+  // software encoders.
+  virtual rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override;
+ protected:
+  ~NativeVideoFrameBuffer() override {}
+ private:
+  int width_;
+  int height_;
+  void *device_pointer_;
+  int pitch_;
+};
+
+
+}  // namespace rigel
+
+#endif  // RIGEL_RTC_CAPTURE_FRAME_H_
+

src/capture_rtc.cc

@@ -2,43 +2,27 @@
 #include <random>
 
 #include "capture_rtc.h"
+#include "capture_frame.h"
 #include "logging.inc"
 
 #include "api/video/i420_buffer.h"
 #include "third_party/libyuv/include/libyuv.h"
 #include "common_video/libyuv/include/webrtc_libyuv.h"
 #include "rtc_base/time_utils.h"
+#include "rtc_base/ref_counted_object.h"
+
 
 namespace rigel {
 
 void VideoCapturer::Initialize() {
   constexpr int width = 960;
   constexpr int height = 544;
   // frame buffer
-  buffer_ = webrtc::I420Buffer::Create(width, height);
-  webrtc::I420Buffer::SetBlack(buffer_.get());
-  // generate frame
-  auto frame = webrtc::VideoFrame::Builder()
-      .set_video_frame_buffer(buffer_)
-      .build();
-  OnFrame(frame);
+  buffer_ = new rtc::RefCountedObject<NativeVideoFrameBuffer>(width, height);
 }
 
 void VideoCapturer::OnRenderFrame(const char *map, int w, int h, int r) {
-  int width = buffer_->width();
-  int height = buffer_->height();
-  const uint8_t *data = reinterpret_cast<const uint8_t *>(map);
-  webrtc::I420Buffer *buffer = buffer_.get();
-  libyuv::ConvertToI420(
-      data, CalcBufferSize(webrtc::VideoType::kARGB, width, height),
-      buffer->MutableDataY(), buffer->StrideY(),
-      buffer->MutableDataU(), buffer->StrideU(),
-      buffer->MutableDataV(), buffer->StrideV(),
-      0, 0,
-      width, height,
-      buffer->width(), buffer->height(),
-      libyuv::kRotate0,
-      libyuv::FOURCC_ABGR);
+  buffer_->Configure((void *)map, r);
   // generate frame
   auto frame = webrtc::VideoFrame::Builder()
       .set_video_frame_buffer(buffer_)

src/capture_rtc.h

@@ -7,6 +7,7 @@
 #include "media/base/video_broadcaster.h"
 #include "api/video/i420_buffer.h"
 #include "render.h"
+#include "capture_frame.h"
 
 namespace rigel {
 
@@ -23,7 +24,7 @@ class VideoCapturer : public rtc::VideoBroadcaster,
   void OnRenderFrame(const char *v, int w, int h, int r) override;
 
  private:
-  rtc::scoped_refptr<webrtc::I420Buffer> buffer_;
+  rtc::scoped_refptr<NativeVideoFrameBuffer> buffer_;
 };
 
 }  // namespace rigel

src/instance.cc

@@ -12,7 +12,7 @@ SignalingInstance::SignalingInstance(
       render_context_(new RenderContext()) {}
 
 void SignalingInstance::Initialize() {
-  rtc_ = new RTCInstance();
+  rtc_ = new RTCInstance(render_context_.get());
 }
 
 SignalingInstance::~SignalingInstance() {

src/instance_rtc.cc

@@ -2,6 +2,7 @@
 #include "instance_rtc.h"
 #include "channel_rtc.h"
 #include "logging.inc"
+#include "render_encoder_factory.h"
 
 #include "api/create_peerconnection_factory.h"
 #include "api/video_codecs/builtin_video_encoder_factory.h"
@@ -19,7 +20,7 @@ static webrtc::PeerConnectionInterface::RTCConfiguration MakeConfiguration() {
 
 namespace rigel {
 
-RTCInstance::RTCInstance() {
+RTCInstance::RTCInstance(RenderContext *render_context) {
   RGL_INFO("Creating RTCInstance");
   // Network Thread
   network_thread_ = rtc::Thread::CreateWithSocketServer();
@@ -38,7 +39,7 @@ RTCInstance::RTCInstance() {
     nullptr,
     webrtc::CreateBuiltinAudioEncoderFactory(),
     webrtc::CreateBuiltinAudioDecoderFactory(),
-    webrtc::CreateBuiltinVideoEncoderFactory(),
+    CreateRigelVideoEncoderFactory(render_context),
     webrtc::CreateBuiltinVideoDecoderFactory(),
     nullptr,
     nullptr);

src/instance_rtc.h

@@ -15,9 +15,11 @@ namespace rigel {
 struct SignalingMessageInterface;
 struct RenderInstanceFactoryInterface;
 
+class RenderContext;
+
 class RTCInstance {
  public:
-  RTCInstance();
+  explicit RTCInstance(RenderContext *render_context);
   ~RTCInstance();
 
   std::unique_ptr<PeerChannelInterface> CreateChannel(

src/render.cc

@@ -2,12 +2,22 @@
 
 #include "render.h"
 #include "render_instance.h"
+#include "render_context_cuda.h"
 
 namespace rigel {
 
+RenderContext::RenderContext() : cuda_(new RenderContextCuda()) {
+
+}
+
+RenderContext::~RenderContext() {
+  delete cuda_;
+  cuda_ = nullptr;
+}
+
 std::unique_ptr<RenderInstanceInterface> RenderContext::CreateInstance(
     RenderInstanceSink *sink) {
-  return std::unique_ptr<RenderInstanceInterface>(new RenderInstance(sink));
+  return std::unique_ptr<RenderInstanceInterface>(new RenderInstance(sink, this));
 }
 
 }  // namespace rigel

src/render.h

@@ -24,12 +24,21 @@ struct RenderInstanceFactoryInterface {
       RenderInstanceSink *sink) = 0;
 };
 
+class RenderContextCuda;
+
 class RenderContext : public RenderInstanceFactoryInterface {
  public:
-  RenderContext() = default;
+  RenderContext();
+  virtual ~RenderContext();
+
   explicit RenderContext(const RenderContext &) = delete;
   std::unique_ptr<RenderInstanceInterface> CreateInstance(
       RenderInstanceSink *sink) override;
+
+  RenderContextCuda *Cuda() const { return cuda_; }
+
+ private:
+  RenderContextCuda *cuda_;
 };
 
 }  // namespace rigel

src/render_context_cuda.cc

@@ -0,0 +1,22 @@
+
+#include "render_context_cuda.h"
+#include "logging.inc"
+
+namespace rigel {
+
+RenderContextCuda::RenderContextCuda() : n_gpu_(0), device_(0), context_(nullptr) {
+  // @see https://github.com/NVIDIA/video-sdk-samples/blob/cf5b2cdbbb0c0869261041289fbadc188ebdc6c0/Samples/AppEncode/AppEncCuda/AppEncCuda.cpp
+  cuInit(0);
+  cuDeviceGetCount(&n_gpu_);
+  cuDeviceGet(&device_, 0);
+  cuDeviceGetName(device_name_, sizeof(device_name_), device_);
+  RGL_INFO("Using CUDA device");
+  RGL_INFO(device_name_);
+  cuCtxCreate(&context_, 0, device_);
+}
+
+RenderContextCuda::~RenderContextCuda() {
+  cuCtxDestroy(context_);
+}
+
+}  // namespace rigel

src/render_context_cuda.h

@@ -0,0 +1,25 @@
+
+#ifndef RIGEL_GRAPHICS_RENDER_CONTEXT_CUDA_H_
+#define RIGEL_GRAPHICS_RENDER_CONTEXT_CUDA_H_
+
+#include <functional>
+#include <cuda.h>
+
+namespace rigel {
+
+class RenderContextCuda {
+ public:
+  RenderContextCuda();
+  ~RenderContextCuda();
+
+  CUcontext Context() const { return context_; }
+ private:
+  int n_gpu_;
+  CUdevice device_;
+  char device_name_[80];
+  CUcontext context_;
+};
+
+}  // namespace rigel
+
+#endif  // RIGEL_GRAPHICS_RENDER_CONTEXT_CUDA_H_