fix for amd video playback (green videos)

src/video_core/host1x/ffmpeg/ffmpeg.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
+// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project & 2024 suyu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #include "common/assert.h"
@@ -218,6 +218,165 @@ bool DecoderContext::OpenContext(const Decoder& decoder) {
     return true;
 }
 
+// Nasty but allows linux builds to pass.
+// Requires double checks when FFMPEG gets updated.
+// Hopefully a future FFMPEG update will all and expose a solution in the public API.
+namespace {
+
+typedef struct FFCodecDefault {
+    const char* key;
+    const char* value;
+} FFCodecDefault;
+
+typedef struct FFCodec {
+    /**
+     * The public AVCodec. See codec.h for it.
+     */
+    AVCodec p;
+
+    /**
+     * Internal codec capabilities FF_CODEC_CAP_*.
+     */
+    unsigned caps_internal : 29;
+
+    /**
+     * This field determines the type of the codec (decoder/encoder)
+     * and also the exact callback cb implemented by the codec.
+     * cb_type uses enum FFCodecType values.
+     */
+    unsigned cb_type : 3;
+
+    int priv_data_size;
+    /**
+     * @name Frame-level threading support functions
+     * @{
+     */
+    /**
+     * Copy necessary context variables from a previous thread context to the current one.
+     * If not defined, the next thread will start automatically; otherwise, the codec
+     * must call ff_thread_finish_setup().
+     *
+     * dst and src will (rarely) point to the same context, in which case memcpy should be skipped.
+     */
+    int (*update_thread_context)(struct AVCodecContext* dst, const struct AVCodecContext* src);
+
+    /**
+     * Copy variables back to the user-facing context
+     */
+    int (*update_thread_context_for_user)(struct AVCodecContext* dst,
+                                          const struct AVCodecContext* src);
+    /** @} */
+
+    /**
+     * Private codec-specific defaults.
+     */
+    const FFCodecDefault* defaults;
+
+    /**
+     * Initialize codec static data, called from av_codec_iterate().
+     *
+     * This is not intended for time consuming operations as it is
+     * run for every codec regardless of that codec being used.
+     */
+    void (*init_static_data)(struct FFCodec* codec);
+
+    int (*init)(struct AVCodecContext*);
+
+    union {
+        /**
+         * Decode to an AVFrame.
+         * cb is in this state if cb_type is FF_CODEC_CB_TYPE_DECODE.
+         *
+         * @param      avctx          codec context
+         * @param[out] frame          AVFrame for output
+         * @param[out] got_frame_ptr  decoder sets to 0 or 1 to indicate that
+         *                            a non-empty frame was returned in frame.
+         * @param[in]  avpkt          AVPacket containing the data to be decoded
+         * @return amount of bytes read from the packet on success,
+         *         negative error code on failure
+         */
+        int (*decode)(struct AVCodecContext* avctx, struct AVFrame* frame, int* got_frame_ptr,
+                      struct AVPacket* avpkt);
+        /**
+         * Decode subtitle data to an AVSubtitle.
+         * cb is in this state if cb_type is FF_CODEC_CB_TYPE_DECODE_SUB.
+         *
+         * Apart from that this is like the decode callback.
+         */
+        int (*decode_sub)(struct AVCodecContext* avctx, struct AVSubtitle* sub, int* got_frame_ptr,
+                          const struct AVPacket* avpkt);
+        /**
+         * Decode API with decoupled packet/frame dataflow.
+         * cb is in this state if cb_type is FF_CODEC_CB_TYPE_RECEIVE_FRAME.
+         *
+         * This function is called to get one output frame. It should call
+         * ff_decode_get_packet() to obtain input data.
+         */
+        int (*receive_frame)(struct AVCodecContext* avctx, struct AVFrame* frame);
+        /**
+         * Encode data to an AVPacket.
+         * cb is in this state if cb_type is FF_CODEC_CB_TYPE_ENCODE
+         *
+         * @param      avctx          codec context
+         * @param[out] avpkt          output AVPacket
+         * @param[in]  frame          AVFrame containing the input to be encoded
+         * @param[out] got_packet_ptr encoder sets to 0 or 1 to indicate that a
+         *                            non-empty packet was returned in avpkt.
+         * @return 0 on success, negative error code on failure
+         */
+        int (*encode)(struct AVCodecContext* avctx, struct AVPacket* avpkt,
+                      const struct AVFrame* frame, int* got_packet_ptr);
+        /**
+         * Encode subtitles to a raw buffer.
+         * cb is in this state if cb_type is FF_CODEC_CB_TYPE_ENCODE_SUB.
+         */
+        int (*encode_sub)(struct AVCodecContext* avctx, uint8_t* buf, int buf_size,
+                          const struct AVSubtitle* sub);
+        /**
+         * Encode API with decoupled frame/packet dataflow.
+         * cb is in this state if cb_type is FF_CODEC_CB_TYPE_RECEIVE_PACKET.
+         *
+         * This function is called to get one output packet.
+         * It should call ff_encode_get_frame() to obtain input data.
+         */
+        int (*receive_packet)(struct AVCodecContext* avctx, struct AVPacket* avpkt);
+    } cb;
+
+    int (*close)(struct AVCodecContext*);
+
+    /**
+     * Flush buffers.
+     * Will be called when seeking
+     */
+    void (*flush)(struct AVCodecContext*);
+
+    /**
+     * Decoding only, a comma-separated list of bitstream filters to apply to
+     * packets before decoding.
+     */
+    const char* bsfs;
+
+    /**
+     * Array of pointers to hardware configurations supported by the codec,
+     * or NULL if no hardware supported.  The array is terminated by a NULL
+     * pointer.
+     *
+     * The user can only access this field via avcodec_get_hw_config().
+     */
+    const struct AVCodecHWConfigInternal* const* hw_configs;
+
+    /**
+     * List of supported codec_tags, terminated by FF_CODEC_TAGS_END.
+     */
+    const uint32_t* codec_tags;
+} FFCodec;
+
+static av_always_inline const FFCodec* ffcodec(const AVCodec* codec) {
+    return (const FFCodec*)codec;
+}
+
+} // namespace
+
 bool DecoderContext::SendPacket(const Packet& packet) {
     m_temp_frame = std::make_shared<Frame>();
     m_got_frame = 0;
@@ -227,8 +386,10 @@ bool DecoderContext::SendPacket(const Packet& packet) {
 #ifndef ANDROID
     if (!m_codec_context->hw_device_ctx && m_codec_context->codec_id == AV_CODEC_ID_H264) {
         m_decode_order = true;
-        const int ret = avcodec_send_frame(m_codec_context, m_temp_frame->GetFrame());
-        if (ret < 0) {
+        auto* codec{ffcodec(m_decoder.GetCodec())};
+        if (const int ret = codec->cb.decode(m_codec_context, m_temp_frame->GetFrame(),
+                                             &m_got_frame, packet.GetPacket());
+            ret < 0) {
             LOG_DEBUG(Service_NVDRV, "avcodec_send_packet error {}", AVError(ret));
             return false;
         }
@@ -250,6 +411,7 @@ std::shared_ptr<Frame> DecoderContext::ReceiveFrame() {
 #ifndef ANDROID
     if (!m_codec_context->hw_device_ctx && m_codec_context->codec_id == AV_CODEC_ID_H264) {
         m_decode_order = true;
+        auto* codec{ffcodec(m_decoder.GetCodec())};
         int ret{0};
 
         if (m_got_frame == 0) {
@@ -257,7 +419,7 @@ std::shared_ptr<Frame> DecoderContext::ReceiveFrame() {
             auto* pkt = packet.GetPacket();
             pkt->data = nullptr;
             pkt->size = 0;
-            ret = avcodec_receive_packet(m_codec_context, pkt);
+            ret = codec->cb.decode(m_codec_context, m_temp_frame->GetFrame(), &m_got_frame, pkt);
             m_codec_context->has_b_frames = 0;
         }