diff --git a/src/main/java/com/gmail/nossr50/listeners/InventoryListener.java b/src/main/java/com/gmail/nossr50/listeners/InventoryListener.java
index 28de0f902..963092ab9 100644
--- a/src/main/java/com/gmail/nossr50/listeners/InventoryListener.java
+++ b/src/main/java/com/gmail/nossr50/listeners/InventoryListener.java
@@ -17,6 +17,7 @@ import org.bukkit.event.inventory.CraftItemEvent;
 import org.bukkit.event.inventory.FurnaceBurnEvent;
 import org.bukkit.event.inventory.FurnaceExtractEvent;
 import org.bukkit.event.inventory.FurnaceSmeltEvent;
+import org.bukkit.event.inventory.InventoryAction;
 import org.bukkit.event.inventory.InventoryClickEvent;
 import org.bukkit.event.inventory.InventoryCloseEvent;
 import org.bukkit.event.inventory.InventoryDragEvent;
@@ -313,6 +314,9 @@ public class InventoryListener implements Listener {
     @EventHandler(priority = EventPriority.MONITOR, ignoreCancelled = true)
     public void onInventoryClickEvent(InventoryClickEvent event) {
         SkillUtils.removeAbilityBuff(event.getCurrentItem());
+        if (event.getAction() == InventoryAction.HOTBAR_SWAP) {
+            SkillUtils.removeAbilityBuff(event.getInventory().getItem(event.getHotbarButton()));
+        }
     }
 
     @EventHandler(priority = EventPriority.MONITOR, ignoreCancelled = true)