[cscore] Use frame time in Linux UsbCameraImpl (wpilibsuite#7609)

cscore/src/main/java/edu/wpi/first/cscore/CvSink.java

@@ -6,6 +6,7 @@
 
 import edu.wpi.first.util.PixelFormat;
 import edu.wpi.first.util.RawFrame;
+import edu.wpi.first.util.TimestampSource;
 import java.nio.ByteBuffer;
 import org.opencv.core.CvType;
 import org.opencv.core.Mat;
@@ -220,4 +221,22 @@ public long grabFrameNoTimeoutDirect() {
     }
     return rv;
   }
+
+  /**
+   * Get the last time a frame was grabbed. This uses the same time base as wpi::Now().
+   *
+   * @return Time in 1 us increments.
+   */
+  public long getLastFrameTime() {
+    return m_frame.getTimestamp();
+  }
+
+  /**
+   * Get the time source for the timestamp the last frame was grabbed at.
+   *
+   * @return Time source
+   */
+  public TimestampSource getLastFrameTimeSource() {
+    return m_frame.getTimestampSource();
+  }
 }

cscore/src/main/native/cpp/Frame.cpp

@@ -16,18 +16,22 @@
 
 using namespace cs;
 
-Frame::Frame(SourceImpl& source, std::string_view error, Time time)
+Frame::Frame(SourceImpl& source, std::string_view error, Time time,
+             WPI_TimestampSource timeSrc)
     : m_impl{source.AllocFrameImpl().release()} {
   m_impl->refcount = 1;
   m_impl->error = error;
   m_impl->time = time;
+  m_impl->timeSource = timeSrc;
 }
 
-Frame::Frame(SourceImpl& source, std::unique_ptr<Image> image, Time time)
+Frame::Frame(SourceImpl& source, std::unique_ptr<Image> image, Time time,
+             WPI_TimestampSource timeSrc)
     : m_impl{source.AllocFrameImpl().release()} {
   m_impl->refcount = 1;
   m_impl->error.resize(0);
   m_impl->time = time;
+  m_impl->timeSource = timeSrc;
   m_impl->images.push_back(image.release());
 }
 

cscore/src/main/native/cpp/Frame.h

@@ -39,6 +39,7 @@ class Frame {
     wpi::recursive_mutex mutex;
     std::atomic_int refcount{0};
     Time time{0};
+    WPI_TimestampSource timeSource{WPI_TIMESRC_UNKNOWN};
     SourceImpl& source;
     std::string error;
     wpi::SmallVector<Image*, 4> images;
@@ -48,9 +49,11 @@ class Frame {
  public:
   Frame() noexcept = default;
 
-  Frame(SourceImpl& source, std::string_view error, Time time);
+  Frame(SourceImpl& source, std::string_view error, Time time,
+        WPI_TimestampSource timeSrc);
 
-  Frame(SourceImpl& source, std::unique_ptr<Image> image, Time time);
+  Frame(SourceImpl& source, std::unique_ptr<Image> image, Time time,
+        WPI_TimestampSource timeSrc);
 
   Frame(const Frame& frame) noexcept : m_impl{frame.m_impl} {
     if (m_impl) {
@@ -75,6 +78,9 @@ class Frame {
   }
 
   Time GetTime() const { return m_impl ? m_impl->time : 0; }
+  WPI_TimestampSource GetTimeSource() const {
+    return m_impl ? m_impl->timeSource : WPI_TIMESRC_UNKNOWN;
+  }
 
   std::string_view GetError() const {
     if (!m_impl) {

cscore/src/main/native/cpp/RawSinkImpl.cpp

@@ -120,6 +120,8 @@ uint64_t RawSinkImpl::GrabFrameImpl(WPI_RawFrame& rawFrame,
   rawFrame.pixelFormat = newImage->pixelFormat;
   rawFrame.size = newImage->size();
   std::copy(newImage->data(), newImage->data() + rawFrame.size, rawFrame.data);
+  rawFrame.timestamp = incomingFrame.GetTime();
+  rawFrame.timestampSrc = incomingFrame.GetTimeSource();
 
   return incomingFrame.GetTime();
 }

cscore/src/main/native/cpp/SourceImpl.cpp

@@ -29,7 +29,7 @@ SourceImpl::SourceImpl(std::string_view name, wpi::Logger& logger,
       m_notifier(notifier),
       m_telemetry(telemetry),
       m_name{name} {
-  m_frame = Frame{*this, std::string_view{}, 0};
+  m_frame = Frame{*this, std::string_view{}, 0, WPI_TIMESRC_UNKNOWN};
 }
 
 SourceImpl::~SourceImpl() {
@@ -95,15 +95,16 @@ Frame SourceImpl::GetNextFrame(double timeout, Frame::Time lastFrameTime) {
   if (!m_frameCv.wait_for(
           lock, std::chrono::milliseconds(static_cast<int>(timeout * 1000)),
           [=, this] { return m_frame.GetTime() != lastFrameTime; })) {
-    m_frame = Frame{*this, "timed out getting frame", wpi::Now()};
+    m_frame = Frame{*this, "timed out getting frame", wpi::Now(),
+                    WPI_TIMESRC_UNKNOWN};
   }
   return m_frame;
 }
 
 void SourceImpl::Wakeup() {
   {
     std::scoped_lock lock{m_frameMutex};
-    m_frame = Frame{*this, std::string_view{}, 0};
+    m_frame = Frame{*this, std::string_view{}, 0, WPI_TIMESRC_UNKNOWN};
   }
   m_frameCv.notify_all();
 }
@@ -463,7 +464,8 @@ std::unique_ptr<Image> SourceImpl::AllocImage(
 }
 
 void SourceImpl::PutFrame(VideoMode::PixelFormat pixelFormat, int width,
-                          int height, std::string_view data, Frame::Time time) {
+                          int height, std::string_view data, Frame::Time time,
+                          WPI_TimestampSource timeSrc) {
   if (pixelFormat == VideoMode::PixelFormat::kBGRA) {
     // Write BGRA as BGR to save a copy
     auto image =
@@ -480,18 +482,19 @@ void SourceImpl::PutFrame(VideoMode::PixelFormat pixelFormat, int width,
           fmt::ptr(data.data()), data.size());
   std::memcpy(image->data(), data.data(), data.size());
 
-  PutFrame(std::move(image), time);
+  PutFrame(std::move(image), time, timeSrc);
 }
 
-void SourceImpl::PutFrame(std::unique_ptr<Image> image, Frame::Time time) {
+void SourceImpl::PutFrame(std::unique_ptr<Image> image, Frame::Time time,
+                          WPI_TimestampSource timeSrc) {
   // Update telemetry
   m_telemetry.RecordSourceFrames(*this, 1);
   m_telemetry.RecordSourceBytes(*this, static_cast<int>(image->size()));
 
   // Update frame
   {
     std::scoped_lock lock{m_frameMutex};
-    m_frame = Frame{*this, std::move(image), time};
+    m_frame = Frame{*this, std::move(image), time, timeSrc};
   }
 
   // Signal listeners
@@ -502,7 +505,7 @@ void SourceImpl::PutError(std::string_view msg, Frame::Time time) {
   // Update frame
   {
     std::scoped_lock lock{m_frameMutex};
-    m_frame = Frame{*this, msg, time};
+    m_frame = Frame{*this, msg, time, WPI_TIMESRC_UNKNOWN};
   }
 
   // Signal listeners

cscore/src/main/native/cpp/SourceImpl.h

@@ -13,6 +13,7 @@
 #include <vector>
 
 #include <wpi/Logger.h>
+#include <wpi/RawFrame.h>
 #include <wpi/condition_variable.h>
 #include <wpi/json_fwd.h>
 #include <wpi/mutex.h>
@@ -141,8 +142,10 @@ class SourceImpl : public PropertyContainer {
                            std::string_view valueStr) override;
 
   void PutFrame(VideoMode::PixelFormat pixelFormat, int width, int height,
-                std::string_view data, Frame::Time time);
-  void PutFrame(std::unique_ptr<Image> image, Frame::Time time);
+                std::string_view data, Frame::Time time,
+                WPI_TimestampSource timeSrc = WPI_TIMESRC_FRAME_DEQUEUE);
+  void PutFrame(std::unique_ptr<Image> image, Frame::Time time,
+                WPI_TimestampSource timeSrc = WPI_TIMESRC_FRAME_DEQUEUE);
   void PutError(std::string_view msg, Frame::Time time);
 
   // Notification functions for corresponding atomics

cscore/src/main/native/include/cscore_cv.h

@@ -8,6 +8,7 @@
 #include <functional>
 
 #include <opencv2/core/mat.hpp>
+#include <wpi/RawFrame.h>
 
 #include "cscore_oo.h"
 #include "cscore_raw.h"
@@ -172,6 +173,23 @@ class CvSink : public ImageSink {
   uint64_t GrabFrameDirectLastTime(cv::Mat& image, uint64_t lastFrameTime,
                                    double timeout = 0.225);
 
+  /**
+   * Get the last time a frame was grabbed. This uses the same time base as
+   * wpi::Now().
+   *
+   * @return Time in 1 us increments.
+   */
+  [[nodiscard]]
+  uint64_t LastFrameTime();
+
+  /**
+   * Get the time source for the timestamp the last frame was grabbed at.
+   *
+   * @return Time source
+   */
+  [[nodiscard]]
+  WPI_TimestampSource LastFrameTimeSource();
+
  private:
   constexpr int GetCvFormat(WPI_PixelFormat pixelFormat);
 
@@ -405,6 +423,14 @@ inline uint64_t CvSink::GrabFrameDirectLastTime(cv::Mat& image,
   return timestamp;
 }
 
+inline uint64_t CvSink::LastFrameTime() {
+  return rawFrame.timestamp;
+}
+
+inline WPI_TimestampSource CvSink::LastFrameTimeSource() {
+  return static_cast<WPI_TimestampSource>(rawFrame.timestampSrc);
+}
+
 }  // namespace cs
 
 #endif  // CSCORE_CSCORE_CV_H_

cscore/src/main/native/linux/UsbCameraImpl.cpp

@@ -555,8 +555,51 @@ void UsbCameraImpl::CameraThreadMain() {
           good = false;
         }
         if (good) {
+          Frame::Time frameTime{wpi::Now()};
+          WPI_TimestampSource timeSource{WPI_TIMESRC_FRAME_DEQUEUE};
+
+          // check the timestamp time
+          auto tsFlags = buf.flags & V4L2_BUF_FLAG_TIMESTAMP_MASK;
+          SDEBUG4("Flags {}", tsFlags);
+          if (tsFlags == V4L2_BUF_FLAG_TIMESTAMP_UNKNOWN) {
+            SDEBUG4("Got unknown time for frame - default to wpi::Now");
+          } else if (tsFlags == V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC) {
+            SDEBUG4("Got valid monotonic time for frame");
+            // we can't go directly to frametime, since the rest of cscore
+            // expects us to use wpi::Now, which is in an arbitrary timebase
+            // (see timestamp.cpp). Best I can do is (approximately) translate
+            // between timebases
+
+            // grab current time in the same timebase as buf.timestamp
+            struct timespec ts;
+            if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) {
+              int64_t nowTime = {ts.tv_sec * 1'000'000 + ts.tv_nsec / 1000};
+              int64_t bufTime = {buf.timestamp.tv_sec * 1'000'000 +
+                                 buf.timestamp.tv_usec};
+              // And offset frameTime by the latency
+              int64_t offset{nowTime - bufTime};
+              frameTime -= offset;
+
+              // Figure out the timestamp's source
+              int tsrcFlags = buf.flags & V4L2_BUF_FLAG_TSTAMP_SRC_MASK;
+              if (tsrcFlags == V4L2_BUF_FLAG_TSTAMP_SRC_EOF) {
+                timeSource = WPI_TIMESRC_V4L_EOF;
+              } else if (tsrcFlags == V4L2_BUF_FLAG_TSTAMP_SRC_SOE) {
+                timeSource = WPI_TIMESRC_V4L_SOE;
+              } else {
+                timeSource = WPI_TIMESRC_UNKNOWN;
+              }
+              SDEBUG4("Frame was {} uS old, flags {}, source {}", offset,
+                      tsrcFlags, static_cast<int>(timeSource));
+            } else {
+              // Can't do anything if we can't access the clock, leave default
+            }
+          } else if (tsFlags == V4L2_BUF_FLAG_TIMESTAMP_COPY) {
+            SDEBUG4("Got valid copy time for frame - default to wpi::Now");
+          }
+
           PutFrame(static_cast<VideoMode::PixelFormat>(m_mode.pixelFormat),
-                   width, height, image, wpi::Now());  // TODO: time
+                   width, height, image, frameTime, timeSource);
         }
       }
 

wpiutil/src/main/java/edu/wpi/first/util/RawFrame.java

@@ -18,6 +18,8 @@ public class RawFrame implements AutoCloseable {
   private int m_height;
   private int m_stride;
   private PixelFormat m_pixelFormat = PixelFormat.kUnknown;
+  private long m_time;
+  private TimestampSource m_timeSource = TimestampSource.kUnknown;
 
   /** Construct a new empty RawFrame. */
   public RawFrame() {
@@ -43,12 +45,15 @@ public void close() {
    * @param stride The number of bytes in each row of image data
    * @param pixelFormat The PixelFormat of the frame
    */
-  void setDataJNI(ByteBuffer data, int width, int height, int stride, int pixelFormat) {
+  void setDataJNI(
+      ByteBuffer data, int width, int height, int stride, int pixelFormat, long time, int timeSrc) {
     m_data = data;
     m_width = width;
     m_height = height;
     m_stride = stride;
     m_pixelFormat = PixelFormat.getFromInt(pixelFormat);
+    m_time = time;
+    m_timeSource = TimestampSource.getFromInt(timeSrc);
   }
 
   /**
@@ -59,11 +64,13 @@ void setDataJNI(ByteBuffer data, int width, int height, int stride, int pixelFor
    * @param stride The number of bytes in each row of image data
    * @param pixelFormat The PixelFormat of the frame
    */
-  void setInfoJNI(int width, int height, int stride, int pixelFormat) {
+  void setInfoJNI(int width, int height, int stride, int pixelFormat, long time, int timeSrc) {
     m_width = width;
     m_height = height;
     m_stride = stride;
     m_pixelFormat = PixelFormat.getFromInt(pixelFormat);
+    m_time = time;
+    m_timeSource = TimestampSource.getFromInt(timeSrc);
   }
 
   /**
@@ -110,6 +117,19 @@ public void setInfo(int width, int height, int stride, PixelFormat pixelFormat)
         pixelFormat.getValue());
   }
 
+  /**
+   * Update this frame's timestamp info.
+   *
+   * @param frameTime the time this frame was grabbed at. This uses the same time base as
+   *     wpi::Now(), in us.
+   * @param frameTimeSource the time source for the timestamp this frame was grabbed at.
+   */
+  public void setTimeInfo(long frameTime, TimestampSource frameTimeSource) {
+    m_time = frameTime;
+    m_timeSource = frameTimeSource;
+    WPIUtilJNI.setRawFrameTime(m_nativeObj, frameTime, frameTimeSource.getValue());
+  }
+
   /**
    * Get the pointer to native representation of this frame.
    *
@@ -185,4 +205,22 @@ public int getStride() {
   public PixelFormat getPixelFormat() {
     return m_pixelFormat;
   }
+
+  /**
+   * Get the time this frame was grabbed at. This uses the same time base as wpi::Now(), in us.
+   *
+   * @return Time in 1 us increments.
+   */
+  public long getTimestamp() {
+    return m_time;
+  }
+
+  /**
+   * Get the time source for the timestamp this frame was grabbed at.
+   *
+   * @return Time source
+   */
+  public TimestampSource getTimestampSource() {
+    return m_timeSource;
+  }
 }