|
|
@@ -3,9 +3,11 @@
|
|
|
// SPDX-License-Identifier: GPL-3.0-or-later Licensed under GPLv3
|
|
|
// or any later version Refer to the license.txt file included.
|
|
|
|
|
|
+#include <bit>
|
|
|
#include <cstdlib>
|
|
|
#include <cstring>
|
|
|
|
|
|
+#include <fmt/format.h>
|
|
|
#include "common/assert.h"
|
|
|
#include "common/logging/log.h"
|
|
|
#include "common/scope_exit.h"
|
|
|
@@ -22,8 +24,19 @@ namespace Service::Nvidia::Devices {
|
|
|
nvhost_ctrl::nvhost_ctrl(Core::System& system_, EventInterface& events_interface_,
|
|
|
NvCore::Container& core_)
|
|
|
: nvdevice{system_}, events_interface{events_interface_}, core{core_},
|
|
|
- syncpoint_manager{core_.GetSyncpointManager()} {}
|
|
|
-nvhost_ctrl::~nvhost_ctrl() = default;
|
|
|
+ syncpoint_manager{core_.GetSyncpointManager()} {
|
|
|
+ events_interface.RegisterForSignal(this);
|
|
|
+}
|
|
|
+
|
|
|
+nvhost_ctrl::~nvhost_ctrl() {
|
|
|
+ events_interface.UnregisterForSignal(this);
|
|
|
+ for (auto& event : events) {
|
|
|
+ if (!event.registered) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ events_interface.FreeEvent(event.kevent);
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
NvResult nvhost_ctrl::Ioctl1(DeviceFD fd, Ioctl command, const std::vector<u8>& input,
|
|
|
std::vector<u8>& output) {
|
|
|
@@ -87,7 +100,7 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
|
|
|
SCOPE_EXIT({
|
|
|
std::memcpy(output.data(), ¶ms, sizeof(params));
|
|
|
if (must_unmark_fail) {
|
|
|
- events_interface.fails[event_id] = 0;
|
|
|
+ events[event_id].fails = 0;
|
|
|
}
|
|
|
});
|
|
|
|
|
|
@@ -116,12 +129,12 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
|
|
|
auto& gpu = system.GPU();
|
|
|
const u32 target_value = params.fence.value;
|
|
|
|
|
|
- auto lock = events_interface.Lock();
|
|
|
+ auto lock = NvEventsLock();
|
|
|
|
|
|
u32 slot = [&]() {
|
|
|
if (is_allocation) {
|
|
|
params.value.raw = 0;
|
|
|
- return events_interface.FindFreeEvent(fence_id);
|
|
|
+ return FindFreeNvEvent(fence_id);
|
|
|
} else {
|
|
|
return params.value.raw;
|
|
|
}
|
|
|
@@ -130,7 +143,7 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
|
|
|
must_unmark_fail = true;
|
|
|
|
|
|
const auto check_failing = [&]() {
|
|
|
- if (events_interface.fails[slot] > 2) {
|
|
|
+ if (events[slot].fails > 2) {
|
|
|
{
|
|
|
auto lk = system.StallProcesses();
|
|
|
gpu.WaitFence(fence_id, target_value);
|
|
|
@@ -142,6 +155,10 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
|
|
|
return false;
|
|
|
};
|
|
|
|
|
|
+ if (slot >= MaxNvEvents) {
|
|
|
+ return NvResult::BadParameter;
|
|
|
+ }
|
|
|
+
|
|
|
if (params.timeout == 0) {
|
|
|
if (check_failing()) {
|
|
|
return NvResult::Success;
|
|
|
@@ -149,17 +166,13 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
|
|
|
return NvResult::Timeout;
|
|
|
}
|
|
|
|
|
|
- if (slot >= MaxNvEvents) {
|
|
|
- return NvResult::BadParameter;
|
|
|
- }
|
|
|
-
|
|
|
- auto* event = events_interface.events[slot];
|
|
|
+ auto& event = events[slot];
|
|
|
|
|
|
- if (!event) {
|
|
|
+ if (!event.registered) {
|
|
|
return NvResult::BadParameter;
|
|
|
}
|
|
|
|
|
|
- if (events_interface.IsBeingUsed(slot)) {
|
|
|
+ if (event.IsBeingUsed()) {
|
|
|
return NvResult::BadParameter;
|
|
|
}
|
|
|
|
|
|
@@ -169,9 +182,9 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
|
|
|
|
|
|
params.value.raw = 0;
|
|
|
|
|
|
- events_interface.status[slot].store(EventState::Waiting, std::memory_order_release);
|
|
|
- events_interface.assigned_syncpt[slot] = fence_id;
|
|
|
- events_interface.assigned_value[slot] = target_value;
|
|
|
+ event.status.store(EventState::Waiting, std::memory_order_release);
|
|
|
+ event.assigned_syncpt = fence_id;
|
|
|
+ event.assigned_value = target_value;
|
|
|
if (is_allocation) {
|
|
|
params.value.syncpoint_id_for_allocation.Assign(static_cast<u16>(fence_id));
|
|
|
params.value.event_allocated.Assign(1);
|
|
|
@@ -189,15 +202,17 @@ NvResult nvhost_ctrl::FreeEvent(u32 slot) {
|
|
|
return NvResult::BadParameter;
|
|
|
}
|
|
|
|
|
|
- if (!events_interface.registered[slot]) {
|
|
|
+ auto& event = events[slot];
|
|
|
+
|
|
|
+ if (!event.registered) {
|
|
|
return NvResult::Success;
|
|
|
}
|
|
|
|
|
|
- if (events_interface.IsBeingUsed(slot)) {
|
|
|
+ if (event.IsBeingUsed()) {
|
|
|
return NvResult::Busy;
|
|
|
}
|
|
|
|
|
|
- events_interface.Free(slot);
|
|
|
+ FreeNvEvent(slot);
|
|
|
return NvResult::Success;
|
|
|
}
|
|
|
|
|
|
@@ -210,15 +225,15 @@ NvResult nvhost_ctrl::IocCtrlEventRegister(const std::vector<u8>& input, std::ve
|
|
|
return NvResult::BadParameter;
|
|
|
}
|
|
|
|
|
|
- auto lock = events_interface.Lock();
|
|
|
+ auto lock = NvEventsLock();
|
|
|
|
|
|
- if (events_interface.registered[event_id]) {
|
|
|
+ if (events[event_id].registered) {
|
|
|
const auto result = FreeEvent(event_id);
|
|
|
if (result != NvResult::Success) {
|
|
|
return result;
|
|
|
}
|
|
|
}
|
|
|
- events_interface.Create(event_id);
|
|
|
+ CreateNvEvent(event_id);
|
|
|
return NvResult::Success;
|
|
|
}
|
|
|
|
|
|
@@ -229,7 +244,7 @@ NvResult nvhost_ctrl::IocCtrlEventUnregister(const std::vector<u8>& input,
|
|
|
const u32 event_id = params.user_event_id & 0x00FF;
|
|
|
LOG_DEBUG(Service_NVDRV, " called, user_event_id: {:X}", event_id);
|
|
|
|
|
|
- auto lock = events_interface.Lock();
|
|
|
+ auto lock = NvEventsLock();
|
|
|
return FreeEvent(event_id);
|
|
|
}
|
|
|
|
|
|
@@ -244,44 +259,121 @@ NvResult nvhost_ctrl::IocCtrlClearEventWait(const std::vector<u8>& input, std::v
|
|
|
return NvResult::BadParameter;
|
|
|
}
|
|
|
|
|
|
- auto lock = events_interface.Lock();
|
|
|
+ auto lock = NvEventsLock();
|
|
|
|
|
|
- if (events_interface.status[event_id].exchange(
|
|
|
- EventState::Cancelling, std::memory_order_acq_rel) == EventState::Waiting) {
|
|
|
- system.GPU().CancelSyncptInterrupt(events_interface.assigned_syncpt[event_id],
|
|
|
- events_interface.assigned_value[event_id]);
|
|
|
- syncpoint_manager.RefreshSyncpoint(events_interface.assigned_syncpt[event_id]);
|
|
|
+ auto& event = events[event_id];
|
|
|
+ if (event.status.exchange(EventState::Cancelling, std::memory_order_acq_rel) ==
|
|
|
+ EventState::Waiting) {
|
|
|
+ system.GPU().CancelSyncptInterrupt(event.assigned_syncpt, event.assigned_value);
|
|
|
+ syncpoint_manager.RefreshSyncpoint(event.assigned_syncpt);
|
|
|
}
|
|
|
- events_interface.fails[event_id]++;
|
|
|
- events_interface.status[event_id].store(EventState::Cancelled, std::memory_order_release);
|
|
|
- events_interface.events[event_id]->GetWritableEvent().Clear();
|
|
|
+ event.fails++;
|
|
|
+ event.status.store(EventState::Cancelled, std::memory_order_release);
|
|
|
+ event.kevent->GetWritableEvent().Clear();
|
|
|
|
|
|
return NvResult::Success;
|
|
|
}
|
|
|
|
|
|
Kernel::KEvent* nvhost_ctrl::QueryEvent(u32 event_id) {
|
|
|
- const auto event = SyncpointEventValue{.raw = event_id};
|
|
|
+ const auto desired_event = SyncpointEventValue{.raw = event_id};
|
|
|
|
|
|
- const bool allocated = event.event_allocated.Value() != 0;
|
|
|
- const u32 slot{allocated ? event.partial_slot.Value() : static_cast<u32>(event.slot)};
|
|
|
+ const bool allocated = desired_event.event_allocated.Value() != 0;
|
|
|
+ const u32 slot{allocated ? desired_event.partial_slot.Value()
|
|
|
+ : static_cast<u32>(desired_event.slot)};
|
|
|
if (slot >= MaxNvEvents) {
|
|
|
ASSERT(false);
|
|
|
return nullptr;
|
|
|
}
|
|
|
|
|
|
- const u32 syncpoint_id{allocated ? event.syncpoint_id_for_allocation.Value()
|
|
|
- : event.syncpoint_id.Value()};
|
|
|
+ const u32 syncpoint_id{allocated ? desired_event.syncpoint_id_for_allocation.Value()
|
|
|
+ : desired_event.syncpoint_id.Value()};
|
|
|
|
|
|
- auto lock = events_interface.Lock();
|
|
|
+ auto lock = NvEventsLock();
|
|
|
|
|
|
- if (events_interface.registered[slot] &&
|
|
|
- events_interface.assigned_syncpt[slot] == syncpoint_id) {
|
|
|
- ASSERT(events_interface.events[slot]);
|
|
|
- return events_interface.events[slot];
|
|
|
+ auto& event = events[slot];
|
|
|
+ if (event.registered && event.assigned_syncpt == syncpoint_id) {
|
|
|
+ ASSERT(event.kevent);
|
|
|
+ return event.kevent;
|
|
|
}
|
|
|
// Is this possible in hardware?
|
|
|
ASSERT_MSG(false, "Slot:{}, SyncpointID:{}, requested", slot, syncpoint_id);
|
|
|
return nullptr;
|
|
|
}
|
|
|
|
|
|
+std::unique_lock<std::mutex> nvhost_ctrl::NvEventsLock() {
|
|
|
+ return std::unique_lock<std::mutex>(events_mutex);
|
|
|
+}
|
|
|
+
|
|
|
+void nvhost_ctrl::CreateNvEvent(u32 event_id) {
|
|
|
+ auto& event = events[event_id];
|
|
|
+ ASSERT(!event.kevent);
|
|
|
+ ASSERT(!event.registered);
|
|
|
+ ASSERT(!event.IsBeingUsed());
|
|
|
+ event.kevent = events_interface.CreateEvent(fmt::format("NVCTRL::NvEvent_{}", event_id));
|
|
|
+ event.status = EventState::Available;
|
|
|
+ event.registered = true;
|
|
|
+ const u64 mask = 1ULL << event_id;
|
|
|
+ event.fails = 0;
|
|
|
+ events_mask |= mask;
|
|
|
+ event.assigned_syncpt = 0;
|
|
|
+}
|
|
|
+
|
|
|
+void nvhost_ctrl::FreeNvEvent(u32 event_id) {
|
|
|
+ auto& event = events[event_id];
|
|
|
+ ASSERT(event.kevent);
|
|
|
+ ASSERT(event.registered);
|
|
|
+ ASSERT(!event.IsBeingUsed());
|
|
|
+ events_interface.FreeEvent(event.kevent);
|
|
|
+ event.kevent = nullptr;
|
|
|
+ event.status = EventState::Available;
|
|
|
+ event.registered = false;
|
|
|
+ const u64 mask = ~(1ULL << event_id);
|
|
|
+ events_mask &= mask;
|
|
|
+}
|
|
|
+
|
|
|
+u32 nvhost_ctrl::FindFreeNvEvent(u32 syncpoint_id) {
|
|
|
+ u32 slot{MaxNvEvents};
|
|
|
+ u32 free_slot{MaxNvEvents};
|
|
|
+ for (u32 i = 0; i < MaxNvEvents; i++) {
|
|
|
+ auto& event = events[i];
|
|
|
+ if (event.registered) {
|
|
|
+ if (!event.IsBeingUsed()) {
|
|
|
+ slot = i;
|
|
|
+ if (event.assigned_syncpt == syncpoint_id) {
|
|
|
+ return slot;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else if (free_slot == MaxNvEvents) {
|
|
|
+ free_slot = i;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (free_slot < MaxNvEvents) {
|
|
|
+ CreateNvEvent(free_slot);
|
|
|
+ return free_slot;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (slot < MaxNvEvents) {
|
|
|
+ return slot;
|
|
|
+ }
|
|
|
+
|
|
|
+ LOG_CRITICAL(Service_NVDRV, "Failed to allocate an event");
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+void nvhost_ctrl::SignalNvEvent(u32 syncpoint_id, u32 value) {
|
|
|
+ const u32 max = MaxNvEvents - std::countl_zero(events_mask);
|
|
|
+ const u32 min = std::countr_zero(events_mask);
|
|
|
+ for (u32 i = min; i < max; i++) {
|
|
|
+ auto& event = events[i];
|
|
|
+ if (event.assigned_syncpt != syncpoint_id || event.assigned_value != value) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (event.status.exchange(EventState::Signalling, std::memory_order_acq_rel) ==
|
|
|
+ EventState::Waiting) {
|
|
|
+ event.kevent->GetWritableEvent().Signal();
|
|
|
+ }
|
|
|
+ event.status.store(EventState::Signalled, std::memory_order_release);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
} // namespace Service::Nvidia::Devices
|