cmake: don't use pkg-config directly with non-reference libusb

CMake Error at externals/libusb/CMakeLists.txt:120 (add_library):
  Cannot find source file:

    libusb/libusb/core.c

  Tried extensions .c .C .c++ .cc .cpp .cxx .cu .mpp .m .M .mm .h .hh .h++
  .hm .hpp .hxx .in .txx .f .F .for .f77 .f90 .f95 .f03 .ispc

CMake Error at externals/libusb/CMakeLists.txt:120 (add_library):
  No SOURCES given to target: usb

ld: error: undefined symbol: libusb_interrupt_transfer
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::SendVibrations()) in archive src/input_common/libinput_common.a
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::GetGCEndpoint(libusb_device*)) in archive src/input_common/libinput_common.a
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::AdapterInputThread()) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_error_name
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::SendVibrations()) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_control_transfer
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::CheckDeviceAccess()) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_kernel_driver_active
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::CheckDeviceAccess()) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_close
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::CheckDeviceAccess()) in archive src/input_common/libinput_common.a
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::ClearLibusbHandle()) in archive src/input_common/libinput_common.a
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::Reset()) in archive src/input_common/libinput_common.a
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::Setup()) in archive src/input_common/libinput_common.a
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::AdapterScanThread()) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_detach_kernel_driver
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::CheckDeviceAccess()) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_claim_interface
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::CheckDeviceAccess()) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_get_config_descriptor
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::GetGCEndpoint(libusb_device*)) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_release_interface
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::ClearLibusbHandle()) in archive src/input_common/libinput_common.a
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::Reset()) in archive src/input_common/libinput_common.a
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::Setup()) in archive src/input_common/libinput_common.a
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::AdapterScanThread()) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_init
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::Adapter()) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_open_device_with_vid_pid
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::Setup()) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_get_device
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::Setup()) in archive src/input_common/libinput_common.a

ld: error: undefined symbol: libusb_exit
>>> referenced by gc_adapter.cpp
>>>               gc_adapter.cpp.o:(GCAdapter::Adapter::Reset()) in archive src/input_common/libinput_common.a

CMakeLists.txt

@@ -424,7 +424,7 @@ endif()
 # Ensure libusb is properly configured (based on dolphin libusb include)
 if(NOT APPLE AND NOT YUZU_USE_BUNDLED_LIBUSB)
     include(FindPkgConfig)
-    if (PKG_CONFIG_FOUND)
+    if (PKG_CONFIG_FOUND AND NOT CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD")
         pkg_check_modules(LIBUSB QUIET libusb-1.0>=1.0.24)
     else()
         find_package(LibUSB)